<commit_before>package main

// reads in a reddit comment archive URL
//  and just extracts the body field
import (
	"bufio"
	"compress/bzip2"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"sync"
)

// globals used for worker queues and global counts
var wg sync.WaitGroup

// Reddit is struct used to unmarshal the reddit comment
type Reddit struct {
	Body string `json:"body"`
}

// doit does the following
//   reads a URL
//   uncompresses it (bzip2)
//   json decodes
//   extracts comment body
//   writes to output file as mini-json
//
//
func doit(prefix, url string) {
	const maxLines = 1 << 31

	// generate outputfile name
	// blah/RC_2015-06.gz --> RC_2015-06-counts.csv
	base := path.Base(url)
	ext := filepath.Ext(base)
	base = base[0:len(base)-len(ext)] + "-body.json"
	log.Printf("[%s] %s -> %s", prefix, url, base)

	resp, err := http.Get(url)
	if err != nil {
		log.Fatalf("[%s] url error: %s", prefix, err)
	}
	defer resp.Body.Close()
	file := bzip2.NewReader(resp.Body)

	// no need to buffer this since raw network and bzip2 will
	// naturally buffer the input
	jsonin := json.NewDecoder(file)

	// set up output file
	fo, err := os.Create(base)
	if err != nil {
		log.Fatalf("[%s] unable to write: %s", prefix, err)
	}
	// buffer output
	bufout := bufio.NewWriter(fo)
	// steam out json
	jsonout := json.NewEncoder(bufout)

	obj := Reddit{}
	lines := 0
	for jsonin.More() && lines < maxLines {
		lines++
		// decode an array value (Message)
		err := jsonin.Decode(&obj)
		if err != nil {
			log.Fatalf("[%s] unable to unmarshal object: %s", prefix, err)
		}
		if obj.Body == "[deleted]" {
			continue
		}
		err = jsonout.Encode(&obj)
		if err != nil {
			log.Fatalf("[%s] unable to marshal object: %s", prefix, err)
		}
	}

	bufout.Flush()
	fo.Close()
	log.Printf("[%s] done %d lines", prefix, lines)
}

func worker(id int, jobs <-chan string) {
	for j := range jobs {
		doit(fmt.Sprintf("%d:%s", id, j), j)
	}
	wg.Done()
}

func main() {

	args := []string{}
	year := 2015
	for month := 1; month <= 12; month++ {
		arg := fmt.Sprintf("http://files.pushshift.io/reddit/comments/RC_%d-%02d.bz2", year, month)
		args = append(args, arg)
	}

	jobs := make(chan string, len(args))

	numCPU := runtime.NumCPU()
	for w := 1; w <= numCPU; w++ {
		wg.Add(1)
		go worker(w, jobs)
	}

	for _, arg := range args {
		log.Printf("[MASTER]: adding %s", arg)
		jobs <- arg
	}
	close(jobs)
	wg.Wait()
	log.Printf("[MASTER]: done")
}
<commit_msg>change to compress output to max out CPU<commit_after>package main

// reads in a reddit comment archive URL
//  and just extracts the body field
import (
	"compress/bzip2"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"sync"
)

// globals used for worker queues and global counts
var wg sync.WaitGroup

// Reddit is struct used to unmarshal the reddit comment
type Reddit struct {
	Body string `json:"body"`
}

// doit does the following
//   reads a URL
//   uncompresses it (bzip2)
//   json decodes
//   extracts comment body
//   writes to output file as mini-json
//
//
func doit(prefix, url string) {
	const maxLines = 1 << 31

	// generate outputfile name
	// blah/RC_2015-06.gz --> RC_2015-06-counts.csv
	base := path.Base(url)
	ext := filepath.Ext(base)
	base = base[0:len(base)-len(ext)] + "-body.json"
	log.Printf("[%s] %s -> %s", prefix, url, base)

	resp, err := http.Get(url)
	if err != nil {
		log.Fatalf("[%s] url error: %s", prefix, err)
	}
	defer resp.Body.Close()
	file := bzip2.NewReader(resp.Body)

	// no need to buffer this since raw network and bzip2 will
	// naturally buffer the input
	jsonin := json.NewDecoder(file)

	// set up output file
	fo, err := os.Create(base)
	if err != nil {
		log.Fatalf("[%s] unable to write: %s", prefix, err)
	}
	// gzip output
	bufout := gzip.NewWriter(fo)
	// steam out json
	jsonout := json.NewEncoder(bufout)

	obj := Reddit{}
	lines := 0
	for jsonin.More() && lines < maxLines {
		lines++
		// decode an array value (Message)
		err := jsonin.Decode(&obj)
		if err != nil {
			log.Fatalf("[%s] unable to unmarshal object: %s", prefix, err)
		}
		if obj.Body == "[deleted]" {
			continue
		}
		err = jsonout.Encode(&obj)
		if err != nil {
			log.Fatalf("[%s] unable to marshal object: %s", prefix, err)
		}
	}

	bufout.Close()
	fo.Close()
	log.Printf("[%s] done %d lines", prefix, lines)
}

func worker(id int, jobs <-chan string) {
	for j := range jobs {
		doit(fmt.Sprintf("%d:%s", id, j), j)
	}
	wg.Done()
}

func main() {

	args := []string{}
	year := 2015
	for month := 1; month <= 12; month++ {
		arg := fmt.Sprintf("http://files.pushshift.io/reddit/comments/RC_%d-%02d.bz2", year, month)
		args = append(args, arg)
	}

	jobs := make(chan string, len(args))

	numCPU := runtime.NumCPU()
	for w := 1; w <= numCPU; w++ {
		wg.Add(1)
		go worker(w, jobs)
	}

	for _, arg := range args {
		log.Printf("[MASTER]: adding %s", arg)
		jobs <- arg
	}
	close(jobs)
	wg.Wait()
	log.Printf("[MASTER]: done")
}
<|endoftext|>


<commit_before>// Copyright (c) 2013-2014 The btcsuite developers
// Copyright (c) 2015-2021 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package blockchain

import (
	"bytes"

	"github.com/decred/dcrd/chaincfg/chainhash"
	"github.com/decred/dcrd/dcrutil/v4"
	"github.com/decred/dcrd/txscript/v4/stdaddr"
	"github.com/decred/dcrd/wire"
)

// lotteryDataForNode is a helper function that returns winning tickets
// along with the ticket pool size and PRNG checksum for a given node.
//
// This function is NOT safe for concurrent access and MUST be called
// with the chainLock held for writes.
func (b *BlockChain) lotteryDataForNode(node *blockNode) ([]chainhash.Hash, int, [6]byte, error) {
	if node.height < b.chainParams.StakeEnabledHeight {
		return nil, 0, [6]byte{}, nil
	}
	stakeNode, err := b.fetchStakeNode(node)
	if err != nil {
		return nil, 0, [6]byte{}, err
	}

	return stakeNode.Winners(), stakeNode.PoolSize(), stakeNode.FinalState(), nil
}

// lotteryDataForBlock takes a node block hash and returns the next tickets
// eligible for voting, the number of tickets in the ticket pool, and the
// final state of the PRNG.
//
// This function is NOT safe for concurrent access and must have the chainLock
// held for write access.
func (b *BlockChain) lotteryDataForBlock(hash *chainhash.Hash) ([]chainhash.Hash, int, [6]byte, error) {
	node := b.index.LookupNode(hash)
	if node == nil {
		return nil, 0, [6]byte{}, unknownBlockError(hash)
	}

	winningTickets, poolSize, finalState, err := b.lotteryDataForNode(node)
	if err != nil {
		return nil, 0, [6]byte{}, err
	}

	return winningTickets, poolSize, finalState, nil
}

// LotteryDataForBlock returns lottery data for a given block in the block
// chain, including side chain blocks.
//
// It is safe for concurrent access.
func (b *BlockChain) LotteryDataForBlock(hash *chainhash.Hash) ([]chainhash.Hash, int, [6]byte, error) {
	b.chainLock.Lock()
	winningTickets, poolSize, finalState, err := b.lotteryDataForBlock(hash)
	b.chainLock.Unlock()
	return winningTickets, poolSize, finalState, err
}

// LiveTickets returns all currently live tickets from the stake database.
//
// This function is safe for concurrent access.
func (b *BlockChain) LiveTickets() ([]chainhash.Hash, error) {
	b.chainLock.RLock()
	sn := b.bestChain.Tip().stakeNode
	b.chainLock.RUnlock()

	return sn.LiveTickets(), nil
}

// TicketsWithAddress returns a slice of ticket hashes that are currently live
// corresponding to the given address.
//
// This function is safe for concurrent access.
func (b *BlockChain) TicketsWithAddress(address stdaddr.StakeAddress) ([]chainhash.Hash, error) {
	b.chainLock.RLock()
	sn := b.bestChain.Tip().stakeNode
	b.chainLock.RUnlock()

	var ticketsWithAddr []chainhash.Hash
	votingRightsScriptVer, votingRightsScript := address.VotingRightsScript()
	tickets := sn.LiveTickets()
	for _, hash := range tickets {
		outpoint := wire.OutPoint{Hash: hash, Index: 0, Tree: wire.TxTreeStake}
		utxo, err := b.utxoCache.FetchEntry(outpoint)
		if err != nil {
			return nil, err
		}

		if utxo.ScriptVersion() == votingRightsScriptVer &&
			bytes.Equal(utxo.PkScript(), votingRightsScript) {

			ticketsWithAddr = append(ticketsWithAddr, hash)
		}
	}

	return ticketsWithAddr, nil
}

// CheckLiveTicket returns whether or not a ticket exists in the live ticket
// treap of the best node.
//
// This function is safe for concurrent access.
func (b *BlockChain) CheckLiveTicket(hash chainhash.Hash) bool {
	b.chainLock.RLock()
	sn := b.bestChain.Tip().stakeNode
	b.chainLock.RUnlock()

	return sn.ExistsLiveTicket(hash)
}

// CheckLiveTickets returns a slice of bools representing whether each ticket
// exists in the live ticket treap of the best node.
//
// This function is safe for concurrent access.
func (b *BlockChain) CheckLiveTickets(hashes []chainhash.Hash) []bool {
	b.chainLock.RLock()
	sn := b.bestChain.Tip().stakeNode
	b.chainLock.RUnlock()

	existsSlice := make([]bool, len(hashes))
	for i := range hashes {
		existsSlice[i] = sn.ExistsLiveTicket(hashes[i])
	}

	return existsSlice
}

// CheckMissedTickets returns a slice of bools representing whether each ticket
// hash has been missed in the live ticket treap of the best node.
//
// This function is safe for concurrent access.
func (b *BlockChain) CheckMissedTickets(hashes []chainhash.Hash) []bool {
	b.chainLock.RLock()
	sn := b.bestChain.Tip().stakeNode
	b.chainLock.RUnlock()

	existsSlice := make([]bool, len(hashes))
	for i := range hashes {
		existsSlice[i] = sn.ExistsMissedTicket(hashes[i])
	}

	return existsSlice
}

// CheckExpiredTicket returns whether or not a ticket was ever expired.
//
// This function is safe for concurrent access.
func (b *BlockChain) CheckExpiredTicket(hash chainhash.Hash) bool {
	b.chainLock.RLock()
	sn := b.bestChain.Tip().stakeNode
	b.chainLock.RUnlock()

	return sn.ExistsExpiredTicket(hash)
}

// CheckExpiredTickets returns whether or not a ticket in a slice of
// tickets was ever expired.
//
// This function is safe for concurrent access.
func (b *BlockChain) CheckExpiredTickets(hashes []chainhash.Hash) []bool {
	b.chainLock.RLock()
	sn := b.bestChain.Tip().stakeNode
	b.chainLock.RUnlock()

	existsSlice := make([]bool, len(hashes))
	for i := range hashes {
		existsSlice[i] = sn.ExistsExpiredTicket(hashes[i])
	}

	return existsSlice
}

// TicketPoolValue returns the current value of all the locked funds in the
// ticket pool.
//
// This function is safe for concurrent access.  All live tickets are at least
// 256 blocks deep on mainnet, so the UTXO set should generally always have
// the asked for transactions.
func (b *BlockChain) TicketPoolValue() (dcrutil.Amount, error) {
	b.chainLock.RLock()
	sn := b.bestChain.Tip().stakeNode
	b.chainLock.RUnlock()

	var amt int64
	for _, hash := range sn.LiveTickets() {
		outpoint := wire.OutPoint{Hash: hash, Index: 0, Tree: wire.TxTreeStake}
		utxo, err := b.utxoCache.FetchEntry(outpoint)
		if err != nil {
			return 0, err
		}

		amt += utxo.Amount()
	}
	return dcrutil.Amount(amt), nil
}
<commit_msg>blockchain: Remove unused CheckMissedTickets.<commit_after>// Copyright (c) 2013-2014 The btcsuite developers
// Copyright (c) 2015-2021 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package blockchain

import (
	"bytes"

	"github.com/decred/dcrd/chaincfg/chainhash"
	"github.com/decred/dcrd/dcrutil/v4"
	"github.com/decred/dcrd/txscript/v4/stdaddr"
	"github.com/decred/dcrd/wire"
)

// lotteryDataForNode is a helper function that returns winning tickets
// along with the ticket pool size and PRNG checksum for a given node.
//
// This function is NOT safe for concurrent access and MUST be called
// with the chainLock held for writes.
func (b *BlockChain) lotteryDataForNode(node *blockNode) ([]chainhash.Hash, int, [6]byte, error) {
	if node.height < b.chainParams.StakeEnabledHeight {
		return nil, 0, [6]byte{}, nil
	}
	stakeNode, err := b.fetchStakeNode(node)
	if err != nil {
		return nil, 0, [6]byte{}, err
	}

	return stakeNode.Winners(), stakeNode.PoolSize(), stakeNode.FinalState(), nil
}

// lotteryDataForBlock takes a node block hash and returns the next tickets
// eligible for voting, the number of tickets in the ticket pool, and the
// final state of the PRNG.
//
// This function is NOT safe for concurrent access and must have the chainLock
// held for write access.
func (b *BlockChain) lotteryDataForBlock(hash *chainhash.Hash) ([]chainhash.Hash, int, [6]byte, error) {
	node := b.index.LookupNode(hash)
	if node == nil {
		return nil, 0, [6]byte{}, unknownBlockError(hash)
	}

	winningTickets, poolSize, finalState, err := b.lotteryDataForNode(node)
	if err != nil {
		return nil, 0, [6]byte{}, err
	}

	return winningTickets, poolSize, finalState, nil
}

// LotteryDataForBlock returns lottery data for a given block in the block
// chain, including side chain blocks.
//
// It is safe for concurrent access.
func (b *BlockChain) LotteryDataForBlock(hash *chainhash.Hash) ([]chainhash.Hash, int, [6]byte, error) {
	b.chainLock.Lock()
	winningTickets, poolSize, finalState, err := b.lotteryDataForBlock(hash)
	b.chainLock.Unlock()
	return winningTickets, poolSize, finalState, err
}

// LiveTickets returns all currently live tickets from the stake database.
//
// This function is safe for concurrent access.
func (b *BlockChain) LiveTickets() ([]chainhash.Hash, error) {
	b.chainLock.RLock()
	sn := b.bestChain.Tip().stakeNode
	b.chainLock.RUnlock()

	return sn.LiveTickets(), nil
}

// TicketsWithAddress returns a slice of ticket hashes that are currently live
// corresponding to the given address.
//
// This function is safe for concurrent access.
func (b *BlockChain) TicketsWithAddress(address stdaddr.StakeAddress) ([]chainhash.Hash, error) {
	b.chainLock.RLock()
	sn := b.bestChain.Tip().stakeNode
	b.chainLock.RUnlock()

	var ticketsWithAddr []chainhash.Hash
	votingRightsScriptVer, votingRightsScript := address.VotingRightsScript()
	tickets := sn.LiveTickets()
	for _, hash := range tickets {
		outpoint := wire.OutPoint{Hash: hash, Index: 0, Tree: wire.TxTreeStake}
		utxo, err := b.utxoCache.FetchEntry(outpoint)
		if err != nil {
			return nil, err
		}

		if utxo.ScriptVersion() == votingRightsScriptVer &&
			bytes.Equal(utxo.PkScript(), votingRightsScript) {

			ticketsWithAddr = append(ticketsWithAddr, hash)
		}
	}

	return ticketsWithAddr, nil
}

// CheckLiveTicket returns whether or not a ticket exists in the live ticket
// treap of the best node.
//
// This function is safe for concurrent access.
func (b *BlockChain) CheckLiveTicket(hash chainhash.Hash) bool {
	b.chainLock.RLock()
	sn := b.bestChain.Tip().stakeNode
	b.chainLock.RUnlock()

	return sn.ExistsLiveTicket(hash)
}

// CheckLiveTickets returns a slice of bools representing whether each ticket
// exists in the live ticket treap of the best node.
//
// This function is safe for concurrent access.
func (b *BlockChain) CheckLiveTickets(hashes []chainhash.Hash) []bool {
	b.chainLock.RLock()
	sn := b.bestChain.Tip().stakeNode
	b.chainLock.RUnlock()

	existsSlice := make([]bool, len(hashes))
	for i := range hashes {
		existsSlice[i] = sn.ExistsLiveTicket(hashes[i])
	}

	return existsSlice
}

// CheckExpiredTicket returns whether or not a ticket was ever expired.
//
// This function is safe for concurrent access.
func (b *BlockChain) CheckExpiredTicket(hash chainhash.Hash) bool {
	b.chainLock.RLock()
	sn := b.bestChain.Tip().stakeNode
	b.chainLock.RUnlock()

	return sn.ExistsExpiredTicket(hash)
}

// CheckExpiredTickets returns whether or not a ticket in a slice of
// tickets was ever expired.
//
// This function is safe for concurrent access.
func (b *BlockChain) CheckExpiredTickets(hashes []chainhash.Hash) []bool {
	b.chainLock.RLock()
	sn := b.bestChain.Tip().stakeNode
	b.chainLock.RUnlock()

	existsSlice := make([]bool, len(hashes))
	for i := range hashes {
		existsSlice[i] = sn.ExistsExpiredTicket(hashes[i])
	}

	return existsSlice
}

// TicketPoolValue returns the current value of all the locked funds in the
// ticket pool.
//
// This function is safe for concurrent access.  All live tickets are at least
// 256 blocks deep on mainnet, so the UTXO set should generally always have
// the asked for transactions.
func (b *BlockChain) TicketPoolValue() (dcrutil.Amount, error) {
	b.chainLock.RLock()
	sn := b.bestChain.Tip().stakeNode
	b.chainLock.RUnlock()

	var amt int64
	for _, hash := range sn.LiveTickets() {
		outpoint := wire.OutPoint{Hash: hash, Index: 0, Tree: wire.TxTreeStake}
		utxo, err := b.utxoCache.FetchEntry(outpoint)
		if err != nil {
			return 0, err
		}

		amt += utxo.Amount()
	}
	return dcrutil.Amount(amt), nil
}
<|endoftext|>


<commit_before>package builder

import (
	"github.com/appc/spec/schema"
	"github.com/blablacar/cnt/cnt"
	"github.com/blablacar/cnt/spec"
	"github.com/blablacar/cnt/utils"
	"github.com/ghodss/yaml"
	"github.com/n0rad/go-erlog/data"
	"github.com/n0rad/go-erlog/errs"
	"github.com/n0rad/go-erlog/logs"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
)

const SH_FUNCTIONS = `
execute_files() {
  fdir=$1
  [ -d "$fdir" ] || return 0

  for file in $fdir/*; do
    [ -e "$file" ] && {
     	[ -x "$file" ] || /cnt/bin/busybox chmod +x "$file"
		isLevelEnabled 4 && echo -e "\e[1m\e[32mRunning script -> $file\e[0m"
     	$file
    }
  done
}

case ` + "`echo ${LOG_LEVEL:-INFO} | awk '{print toupper($0)}'`" + ` in
	"FATAL") lvl=0 ;;
	"PANIC") lvl=1 ;;
	"ERROR") lvl=2 ;;
	"WARN"|"WARNING") lvl=3 ;;
	"INFO") lvl=4 ;;
	"DEBUG") lvl=5 ;;
	"TRACE") lvl=6 ;;
	*) echo "UNKNOWN LOG LEVEL"; lvl=4 ;;
esac

isLevelEnabled() {
	if [ $1 -le $lvl ]; then
		return 0
	fi
	return 1
}
`

const BUILD_SCRIPT = `#!/cnt/bin/busybox sh
set -e
` + SH_FUNCTIONS + `

isLevelEnabled 5 && set -x

export TARGET=$( dirname $0 )
export ROOTFS=%%ROOTFS%%
export TERM=xterm

execute_files "$ROOTFS/cnt/runlevels/inherit-build-early"
execute_files "$TARGET/runlevels/build"
`

const BUILD_SCRIPT_LATE = `#!/cnt/bin/busybox sh
set -e
` + SH_FUNCTIONS + `

isLevelEnabled 5 && set -x

export TARGET=$( dirname $0 )
export ROOTFS=%%ROOTFS%%
export TERM=xterm

execute_files "$TARGET/runlevels/build-late"
execute_files "$ROOTFS/cnt/runlevels/inherit-build-late"
`

const PRESTART = `#!/cnt/bin/busybox sh
set -e
` + SH_FUNCTIONS + `

isLevelEnabled 5 && set -x

BASEDIR=${0%/*}
CNT_PATH=/cnt

execute_files ${CNT_PATH}/runlevels/prestart-early

if [ -z ${LOG_LEVEL} ]; then
	${BASEDIR}/templater -o TEMPLATER_OVERRIDE -t / /cnt
else
	${BASEDIR}/templater -o TEMPLATER_OVERRIDE -L "${LOG_LEVEL}" -t / /cnt
fi

#if [ -d ${CNT_PATH}/attributes ]; then
#	echo "$CONFD_OVERRIDE"
#    ${BASEDIR}/attributes-merger -i ${CNT_PATH}/attributes -e CONFD_OVERRIDE
#    export CONFD_DATA=$(cat attributes.json)
#fi
#${BASEDIR}/confd -onetime -config-file=${CNT_PATH}/prestart/confd.toml

execute_files ${CNT_PATH}/runlevels/prestart-late
`
const PATH_BIN = "/bin"
const PATH_TESTS = "/tests"
const PATH_INSTALLED = "/installed"
const PATH_MANIFEST = "/manifest"
const PATH_IMAGE_ACI = "/image.aci"
const PATH_IMAGE_ACI_ZIP = "/image-zip.aci"
const PATH_ROOTFS = "/rootfs"
const PATH_TARGET = "/target"
const PATH_CNT = "/cnt"
const PATH_CNT_MANIFEST = "/cnt-manifest.yml"
const PATH_RUNLEVELS = "/runlevels"
const PATH_PRESTART_EARLY = "/prestart-early"
const PATH_PRESTART_LATE = "/prestart-late"
const PATH_INHERIT_BUILD_LATE = "/inherit-build-late"
const PATH_INHERIT_BUILD_EARLY = "/inherit-build-early"
const PATH_ATTRIBUTES = "/attributes"
const PATH_FILES = "/files"
const PATH_BUILD_LATE = "/build-late"
const PATH_BUILD_SETUP = "/build-setup"
const PATH_BUILD = "/build"
const PATH_TEMPLATES = "/templates"

type Aci struct {
	fields          data.Fields
	path            string
	target          string
	rootfs          string
	podName         *spec.ACFullname
	manifest        spec.AciManifest
	args            BuildArgs
	FullyResolveDep bool
}

func NewAciWithManifest(path string, args BuildArgs, manifest spec.AciManifest, latestChecked *chan bool, compatChecked *chan bool) (*Aci, error) {
	if manifest.NameAndVersion == "" {
		logs.WithField("path", path).Fatal("name is mandatory in manifest")
	}

	fields := data.WithField("aci", manifest.NameAndVersion.String())
	logs.WithF(fields).WithFields(data.Fields{"args": args, "path": path, "manifest": manifest}).Debug("New aci")

	fullPath, err := filepath.Abs(path)
	if err != nil {
		return nil, errs.WithEF(err, fields, "Cannot get fullpath of project")
	}

	target := fullPath + PATH_TARGET
	if cnt.Home.Config.TargetWorkDir != "" {
		currentAbsDir, err := filepath.Abs(cnt.Home.Config.TargetWorkDir + "/" + manifest.NameAndVersion.ShortName())
		if err != nil {
			return nil, errs.WithEF(err, fields.WithField("path", path), "Invalid target path")
		}
		target = currentAbsDir
	}

	aci := &Aci{
		fields:          fields,
		args:            args,
		path:            fullPath,
		manifest:        manifest,
		target:          target,
		rootfs:          target + PATH_ROOTFS,
		FullyResolveDep: true,
	}

	go aci.checkCompatibilityVersions(compatChecked)
	go aci.checkLatestVersions(latestChecked)
	return aci, nil
}

func NewAci(path string, args BuildArgs) (*Aci, error) {
	manifest, err := readAciManifest(path + PATH_CNT_MANIFEST)
	if err != nil {
		return nil, errs.WithEF(err, data.WithField("path", path+PATH_CNT_MANIFEST), "Cannot read manifest")
	}
	return NewAciWithManifest(path, args, *manifest, nil, nil)
}

//////////////////////////////////////////////////////////////////

func readAciManifest(manifestPath string) (*spec.AciManifest, error) {
	manifest := spec.AciManifest{Aci: spec.AciDefinition{}}

	source, err := ioutil.ReadFile(manifestPath)
	if err != nil {
		return nil, err
	}
	err = yaml.Unmarshal([]byte(source), &manifest)
	if err != nil {
		return nil, errs.WithE(err, "Cannot unmarshall manifest")
	}

	return &manifest, nil
}

func (aci *Aci) tarAci(zip bool) {
	target := PATH_IMAGE_ACI[1:]
	if zip {
		target = PATH_IMAGE_ACI_ZIP[1:]
	}
	dir, _ := os.Getwd()
	logs.WithField("path", aci.target).Debug("chdir")
	os.Chdir(aci.target)
	utils.Tar(zip, target, PATH_MANIFEST[1:], PATH_ROOTFS[1:])
	logs.WithField("path", dir).Debug("chdir")
	os.Chdir(dir)
}

func (aci *Aci) checkCompatibilityVersions(compatChecked *chan bool) {
	froms, err := aci.manifest.GetFroms()
	if err != nil {
		logs.WithEF(err, aci.fields).Fatal("Invalid from")
	}
	for _, from := range froms {
		if from == "" {
			continue
		}

		fromFields := aci.fields.WithField("dependency", from.String())

		err := utils.ExecCmd("rkt", "--insecure-options=image", "fetch", from.String())
		if err != nil {
			logs.WithEF(err, fromFields).Fatal("Cannot fetch from")
		}
		out, err := utils.ExecCmdGetOutput("rkt", "image", "cat-manifest", from.String())
		if err != nil {
			logs.WithEF(err, fromFields).Fatal("Cannot find dependency")
		}

		version, ok := loadManifest(out).Annotations.Get("cnt-version")
		var val int
		if ok {
			val, err = strconv.Atoi(version)
			if err != nil {
				logs.WithEF(err, fromFields).WithField("version", version).Fatal("Failed to parse cnt-version from manifest")
			}
		}
		if !ok || val < 51 {
			logs.WithF(aci.fields).
				WithField("from", from).
				WithField("require", ">=51").
				Error("from aci was not build with a compatible version of cnt")
		}
	}

	for _, dep := range aci.manifest.Aci.Dependencies {
		depFields := aci.fields.WithField("dependency", dep.String())
		err := utils.ExecCmd("rkt", "--insecure-options=image", "fetch", dep.String())
		if err != nil {
			logs.WithEF(err, depFields).Fatal("Cannot fetch dependency")
		}
		out, err := utils.ExecCmdGetOutput("rkt", "image", "cat-manifest", dep.String())
		if err != nil {
			logs.WithEF(err, depFields).Fatal("Cannot find dependency")
		}

		version, ok := loadManifest(out).Annotations.Get("cnt-version")
		var val int
		if ok {
			val, err = strconv.Atoi(version)
			if err != nil {
				logs.WithEF(err, depFields).WithField("version", version).Fatal("Failed to parse cnt-version from manifest")
			}
		}
		if !ok || val < 51 {
			logs.WithF(aci.fields).
				WithField("dependency", dep).
				WithField("require", ">=51").
				Error("dependency aci was not build with a compatible version of cnt")
		}
	}
	if compatChecked != nil {
		*compatChecked <- true
	}
}

func loadManifest(content string) schema.ImageManifest {
	im := schema.ImageManifest{}
	err := im.UnmarshalJSON([]byte(content))
	if err != nil {
		logs.WithE(err).WithField("content", content).Fatal("Failed to read manifest content")
	}
	return im
}

func (aci *Aci) checkLatestVersions(latestChecked *chan bool) {
	froms, err := aci.manifest.GetFroms()
	if err != nil {
		logs.WithEF(err, aci.fields).Fatal("Invalid from")
	}
	for _, from := range froms {
		if from == "" {
			continue
		}

		version, _ := from.LatestVersion()
		logs.WithField("version", from.Name()+":"+version).Debug("Discovered from latest verion")
		if version != "" && utils.Version(from.Version()).LessThan(utils.Version(version)) {
			logs.WithF(aci.fields.WithField("version", from.Name()+":"+version)).Warn("Newer 'from' version")
		}
	}
	for _, dep := range aci.manifest.Aci.Dependencies {
		if dep.Version() == "" {
			continue
		}
		version, _ := dep.LatestVersion()
		if version != "" && utils.Version(dep.Version()).LessThan(utils.Version(version)) {
			logs.WithF(aci.fields.WithField("version", dep.Name()+":"+version)).Warn("Newer 'dependency' version")
		}
	}
	if latestChecked != nil {
		*latestChecked <- true
	}
}
<commit_msg>remove goroutine to check version and compatibility due to rkt bug<commit_after>package builder

import (
	"github.com/appc/spec/schema"
	"github.com/blablacar/cnt/cnt"
	"github.com/blablacar/cnt/spec"
	"github.com/blablacar/cnt/utils"
	"github.com/ghodss/yaml"
	"github.com/n0rad/go-erlog/data"
	"github.com/n0rad/go-erlog/errs"
	"github.com/n0rad/go-erlog/logs"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
)

const SH_FUNCTIONS = `
execute_files() {
  fdir=$1
  [ -d "$fdir" ] || return 0

  for file in $fdir/*; do
    [ -e "$file" ] && {
     	[ -x "$file" ] || /cnt/bin/busybox chmod +x "$file"
		isLevelEnabled 4 && echo -e "\e[1m\e[32mRunning script -> $file\e[0m"
     	$file
    }
  done
}

case ` + "`echo ${LOG_LEVEL:-INFO} | awk '{print toupper($0)}'`" + ` in
	"FATAL") lvl=0 ;;
	"PANIC") lvl=1 ;;
	"ERROR") lvl=2 ;;
	"WARN"|"WARNING") lvl=3 ;;
	"INFO") lvl=4 ;;
	"DEBUG") lvl=5 ;;
	"TRACE") lvl=6 ;;
	*) echo "UNKNOWN LOG LEVEL"; lvl=4 ;;
esac

isLevelEnabled() {
	if [ $1 -le $lvl ]; then
		return 0
	fi
	return 1
}
`

const BUILD_SCRIPT = `#!/cnt/bin/busybox sh
set -e
` + SH_FUNCTIONS + `

isLevelEnabled 5 && set -x

export TARGET=$( dirname $0 )
export ROOTFS=%%ROOTFS%%
export TERM=xterm

execute_files "$ROOTFS/cnt/runlevels/inherit-build-early"
execute_files "$TARGET/runlevels/build"
`

const BUILD_SCRIPT_LATE = `#!/cnt/bin/busybox sh
set -e
` + SH_FUNCTIONS + `

isLevelEnabled 5 && set -x

export TARGET=$( dirname $0 )
export ROOTFS=%%ROOTFS%%
export TERM=xterm

execute_files "$TARGET/runlevels/build-late"
execute_files "$ROOTFS/cnt/runlevels/inherit-build-late"
`

const PRESTART = `#!/cnt/bin/busybox sh
set -e
` + SH_FUNCTIONS + `

isLevelEnabled 5 && set -x

BASEDIR=${0%/*}
CNT_PATH=/cnt

execute_files ${CNT_PATH}/runlevels/prestart-early

if [ -z ${LOG_LEVEL} ]; then
	${BASEDIR}/templater -o TEMPLATER_OVERRIDE -t / /cnt
else
	${BASEDIR}/templater -o TEMPLATER_OVERRIDE -L "${LOG_LEVEL}" -t / /cnt
fi

#if [ -d ${CNT_PATH}/attributes ]; then
#	echo "$CONFD_OVERRIDE"
#    ${BASEDIR}/attributes-merger -i ${CNT_PATH}/attributes -e CONFD_OVERRIDE
#    export CONFD_DATA=$(cat attributes.json)
#fi
#${BASEDIR}/confd -onetime -config-file=${CNT_PATH}/prestart/confd.toml

execute_files ${CNT_PATH}/runlevels/prestart-late
`
const PATH_BIN = "/bin"
const PATH_TESTS = "/tests"
const PATH_INSTALLED = "/installed"
const PATH_MANIFEST = "/manifest"
const PATH_IMAGE_ACI = "/image.aci"
const PATH_IMAGE_ACI_ZIP = "/image-zip.aci"
const PATH_ROOTFS = "/rootfs"
const PATH_TARGET = "/target"
const PATH_CNT = "/cnt"
const PATH_CNT_MANIFEST = "/cnt-manifest.yml"
const PATH_RUNLEVELS = "/runlevels"
const PATH_PRESTART_EARLY = "/prestart-early"
const PATH_PRESTART_LATE = "/prestart-late"
const PATH_INHERIT_BUILD_LATE = "/inherit-build-late"
const PATH_INHERIT_BUILD_EARLY = "/inherit-build-early"
const PATH_ATTRIBUTES = "/attributes"
const PATH_FILES = "/files"
const PATH_BUILD_LATE = "/build-late"
const PATH_BUILD_SETUP = "/build-setup"
const PATH_BUILD = "/build"
const PATH_TEMPLATES = "/templates"

type Aci struct {
	fields          data.Fields
	path            string
	target          string
	rootfs          string
	podName         *spec.ACFullname
	manifest        spec.AciManifest
	args            BuildArgs
	FullyResolveDep bool
}

func NewAciWithManifest(path string, args BuildArgs, manifest spec.AciManifest, latestChecked *chan bool, compatChecked *chan bool) (*Aci, error) {
	if manifest.NameAndVersion == "" {
		logs.WithField("path", path).Fatal("name is mandatory in manifest")
	}

	fields := data.WithField("aci", manifest.NameAndVersion.String())
	logs.WithF(fields).WithFields(data.Fields{"args": args, "path": path, "manifest": manifest}).Debug("New aci")

	fullPath, err := filepath.Abs(path)
	if err != nil {
		return nil, errs.WithEF(err, fields, "Cannot get fullpath of project")
	}

	target := fullPath + PATH_TARGET
	if cnt.Home.Config.TargetWorkDir != "" {
		currentAbsDir, err := filepath.Abs(cnt.Home.Config.TargetWorkDir + "/" + manifest.NameAndVersion.ShortName())
		if err != nil {
			return nil, errs.WithEF(err, fields.WithField("path", path), "Invalid target path")
		}
		target = currentAbsDir
	}

	aci := &Aci{
		fields:          fields,
		args:            args,
		path:            fullPath,
		manifest:        manifest,
		target:          target,
		rootfs:          target + PATH_ROOTFS,
		FullyResolveDep: true,
	}

	aci.checkCompatibilityVersions(compatChecked)
	aci.checkLatestVersions(latestChecked)
	return aci, nil
}

func NewAci(path string, args BuildArgs) (*Aci, error) {
	manifest, err := readAciManifest(path + PATH_CNT_MANIFEST)
	if err != nil {
		return nil, errs.WithEF(err, data.WithField("path", path+PATH_CNT_MANIFEST), "Cannot read manifest")
	}
	return NewAciWithManifest(path, args, *manifest, nil, nil)
}

//////////////////////////////////////////////////////////////////

func readAciManifest(manifestPath string) (*spec.AciManifest, error) {
	manifest := spec.AciManifest{Aci: spec.AciDefinition{}}

	source, err := ioutil.ReadFile(manifestPath)
	if err != nil {
		return nil, err
	}
	err = yaml.Unmarshal([]byte(source), &manifest)
	if err != nil {
		return nil, errs.WithE(err, "Cannot unmarshall manifest")
	}

	return &manifest, nil
}

func (aci *Aci) tarAci(zip bool) {
	target := PATH_IMAGE_ACI[1:]
	if zip {
		target = PATH_IMAGE_ACI_ZIP[1:]
	}
	dir, _ := os.Getwd()
	logs.WithField("path", aci.target).Debug("chdir")
	os.Chdir(aci.target)
	utils.Tar(zip, target, PATH_MANIFEST[1:], PATH_ROOTFS[1:])
	logs.WithField("path", dir).Debug("chdir")
	os.Chdir(dir)
}

func (aci *Aci) checkCompatibilityVersions(compatChecked *chan bool) {
	froms, err := aci.manifest.GetFroms()
	if err != nil {
		logs.WithEF(err, aci.fields).Fatal("Invalid from")
	}
	for _, from := range froms {
		if from == "" {
			continue
		}

		fromFields := aci.fields.WithField("dependency", from.String())

		err := utils.ExecCmd("rkt", "--insecure-options=image", "fetch", from.String())
		if err != nil {
			logs.WithEF(err, fromFields).Fatal("Cannot fetch from")
		}
		out, err := utils.ExecCmdGetOutput("rkt", "image", "cat-manifest", from.String())
		if err != nil {
			logs.WithEF(err, fromFields).Fatal("Cannot find dependency")
		}

		version, ok := loadManifest(out).Annotations.Get("cnt-version")
		var val int
		if ok {
			val, err = strconv.Atoi(version)
			if err != nil {
				logs.WithEF(err, fromFields).WithField("version", version).Fatal("Failed to parse cnt-version from manifest")
			}
		}
		if !ok || val < 51 {
			logs.WithF(aci.fields).
				WithField("from", from).
				WithField("require", ">=51").
				Error("from aci was not build with a compatible version of cnt")
		}
	}

	for _, dep := range aci.manifest.Aci.Dependencies {
		depFields := aci.fields.WithField("dependency", dep.String())
		err := utils.ExecCmd("rkt", "--insecure-options=image", "fetch", dep.String())
		if err != nil {
			logs.WithEF(err, depFields).Fatal("Cannot fetch dependency")
		}
		out, err := utils.ExecCmdGetOutput("rkt", "image", "cat-manifest", dep.String())
		if err != nil {
			logs.WithEF(err, depFields).Fatal("Cannot find dependency")
		}

		version, ok := loadManifest(out).Annotations.Get("cnt-version")
		var val int
		if ok {
			val, err = strconv.Atoi(version)
			if err != nil {
				logs.WithEF(err, depFields).WithField("version", version).Fatal("Failed to parse cnt-version from manifest")
			}
		}
		if !ok || val < 51 {
			logs.WithF(aci.fields).
				WithField("dependency", dep).
				WithField("require", ">=51").
				Error("dependency aci was not build with a compatible version of cnt")
		}
	}
	if compatChecked != nil {
		*compatChecked <- true
	}
}

func loadManifest(content string) schema.ImageManifest {
	im := schema.ImageManifest{}
	err := im.UnmarshalJSON([]byte(content))
	if err != nil {
		logs.WithE(err).WithField("content", content).Fatal("Failed to read manifest content")
	}
	return im
}

func (aci *Aci) checkLatestVersions(latestChecked *chan bool) {
	froms, err := aci.manifest.GetFroms()
	if err != nil {
		logs.WithEF(err, aci.fields).Fatal("Invalid from")
	}
	for _, from := range froms {
		if from == "" {
			continue
		}

		version, _ := from.LatestVersion()
		logs.WithField("version", from.Name()+":"+version).Debug("Discovered from latest verion")
		if version != "" && utils.Version(from.Version()).LessThan(utils.Version(version)) {
			logs.WithF(aci.fields.WithField("version", from.Name()+":"+version)).Warn("Newer 'from' version")
		}
	}
	for _, dep := range aci.manifest.Aci.Dependencies {
		if dep.Version() == "" {
			continue
		}
		version, _ := dep.LatestVersion()
		if version != "" && utils.Version(dep.Version()).LessThan(utils.Version(version)) {
			logs.WithF(aci.fields.WithField("version", dep.Name()+":"+version)).Warn("Newer 'dependency' version")
		}
	}
	if latestChecked != nil {
		*latestChecked <- true
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Knative Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
	"github.com/knative/pkg/apis"
	duckv1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/intstr"
)

// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// +genclient:nonNamespaced

// ClusterIngress is a collection of rules that allow inbound connections to reach the
// endpoints defined by a backend. An ClusterIngress can be configured to give services
// externally-reachable urls, load balance traffic offer name based virtual hosting etc.
//
// This is heavily based on K8s Ingress https://godoc.org/k8s.io/api/extensions/v1beta1#Ingress
// which some highlighted modifications.
type ClusterIngress struct {
	metav1.TypeMeta `json:",inline"`
	// Standard object's metadata.
	// More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata
	// +optional
	metav1.ObjectMeta `json:"metadata,omitempty"`

	// Spec is the desired state of the ClusterIngress.
	// More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status
	// +optional
	Spec IngressSpec `json:"spec,omitempty"`

	// Status is the current state of the ClusterIngress.
	// More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status
	// +optional
	Status IngressStatus `json:"status,omitempty"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// ClusterIngressList is a collection of ClusterIngress.
type ClusterIngressList struct {
	metav1.TypeMeta `json:",inline"`
	// Standard object's metadata.
	// More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata
	// +optional
	metav1.ListMeta `json:"metadata,omitempty"`

	// Items is the list of ClusterIngress.
	Items []ClusterIngress `json:"items"`
}

// IngressSpec describes the ClusterIngress the user wishes to exist.
//
// In general this follow the same shape as K8s Ingress.  Some notable differences:
// - Backends now can have namespace:
// - Traffic can be split across multiple backends.
// - Timeout & Retry can be configured.
// - Headers can be appended.
type IngressSpec struct {
	// DeprecatedGeneration was used prior in Kubernetes versions <1.11
	// when metadata.generation was not being incremented by the api server
	//
	// This property will be dropped in future Knative releases and should
	// not be used - use metadata.generation
	//
	// Tracking issue: https://github.com/knative/serving/issues/643
	//
	// +optional
	DeprecatedGeneration int64 `json:"generation,omitempty"`

	// TLS configuration. Currently the ClusterIngress only supports a single TLS
	// port, 443. If multiple members of this list specify different hosts, they
	// will be multiplexed on the same port according to the hostname specified
	// through the SNI TLS extension, if the ingress controller fulfilling the
	// ingress supports SNI.
	// +optional
	TLS []ClusterIngressTLS `json:"tls,omitempty"`

	// A list of host rules used to configure the ClusterIngress.
	// +optional
	Rules []ClusterIngressRule `json:"rules,omitempty"`

	// Visibility setting.
	Visibility IngressVisibility `json:"visibility,omitempty"`
}

// IngressVisibility describes whether the Ingress should be exposed to
// public gateways or not.
type IngressVisibility string

const (
	// IngressVisibilityExternalIP is used to denote that the Ingress
	// should be exposed to an external IP, for example a LoadBalancer
	// Service.  This is the default value for IngressVisibility.
	IngressVisibilityExternalIP IngressVisibility = "ExternalIP"
	// IngressVisibilityClusterLocal is used to denote that the Ingress
	// should be only be exposed locally to the cluster.
	IngressVisibilityClusterLocal IngressVisibility = "ClusterLocal"
)

// ClusterIngressTLS describes the transport layer security associated with an ClusterIngress.
type ClusterIngressTLS struct {
	// Hosts are a list of hosts included in the TLS certificate. The values in
	// this list must match the name/s used in the tlsSecret. Defaults to the
	// wildcard host setting for the loadbalancer controller fulfilling this
	// ClusterIngress, if left unspecified.
	// +optional
	Hosts []string `json:"hosts,omitempty"`

	// SecretName is the name of the secret used to terminate SSL traffic.
	SecretName string `json:"secretName,omitempty"`

	// SecretNamespace is the namespace of the secret used to terminate SSL traffic.
	SecretNamespace string `json:"secretNamespace,omitempty"`

	// ServerCertificate identifies the certificate filename in the secret.
	// Defaults to `tls.cert`.
	// +optional
	ServerCertificate string `json:"serverCertificate,omitempty"`

	// PrivateKey identifies the private key filename in the secret.
	// Defaults to `tls.key`.
	// +optional
	PrivateKey string `json:"privateKey,omitempty"`
}

// ClusterIngressRule represents the rules mapping the paths under a specified host to
// the related backend services. Incoming requests are first evaluated for a host
// match, then routed to the backend associated with the matching ClusterIngressRuleValue.
type ClusterIngressRule struct {
	// Host is the fully qualified domain name of a network host, as defined
	// by RFC 3986. Note the following deviations from the "host" part of the
	// URI as defined in the RFC:
	// 1. IPs are not allowed. Currently a rule value can only apply to the
	//	  IP in the Spec of the parent ClusterIngress.
	// 2. The `:` delimiter is not respected because ports are not allowed.
	//	  Currently the port of an ClusterIngress is implicitly :80 for http and
	//	  :443 for https.
	// Both these may change in the future.
	// If the host is unspecified, the ClusterIngress routes all traffic based on the
	// specified ClusterIngressRuleValue.
	// If multiple matching Hosts were provided, the first rule will take precedent.
	// +optional
	Hosts []string `json:"hosts,omitempty"`

	// HTTP represents a rule to apply against incoming requests. If the
	// rule is satisfied, the request is routed to the specified backend.
	HTTP *HTTPClusterIngressRuleValue `json:"http,omitempty"`
}

// HTTPClusterIngressRuleValue is a list of http selectors pointing to backends.
// In the example: http://<host>/<path>?<searchpart> -> backend where
// where parts of the url correspond to RFC 3986, this resource will be used
// to match against everything after the last '/' and before the first '?'
// or '#'.
type HTTPClusterIngressRuleValue struct {
	// A collection of paths that map requests to backends.
	//
	// If they are multiple matching paths, the first match takes precendent.
	Paths []HTTPClusterIngressPath `json:"paths"`

	// TODO: Consider adding fields for ingress-type specific global
	// options usable by a loadbalancer, like http keep-alive.
}

// HTTPClusterIngressPath associates a path regex with a backend. Incoming urls matching
// the path are forwarded to the backend.
type HTTPClusterIngressPath struct {
	// Path is an extended POSIX regex as defined by IEEE Std 1003.1,
	// (i.e this follows the egrep/unix syntax, not the perl syntax)
	// matched against the path of an incoming request. Currently it can
	// contain characters disallowed from the conventional "path"
	// part of a URL as defined by RFC 3986. Paths must begin with
	// a '/'. If unspecified, the path defaults to a catch all sending
	// traffic to the backend.
	// +optional
	Path string `json:"path,omitempty"`

	// Splits defines the referenced service endpoints to which the traffic
	// will be forwarded to.
	Splits []ClusterIngressBackendSplit `json:"splits"`

	// AppendHeaders allow specifying additional HTTP headers to add
	// before forwarding a request to the destination service.
	//
	// NOTE: This differs from K8s Ingress which doesn't allow header appending.
	// +optional
	AppendHeaders map[string]string `json:"appendHeaders,omitempty"`

	// Timeout for HTTP requests.
	//
	// NOTE: This differs from K8s Ingress which doesn't allow setting timeouts.
	// +optional
	Timeout *metav1.Duration `json:"timeout,omitempty"`

	// Retry policy for HTTP requests.
	//
	// NOTE: This differs from K8s Ingress which doesn't allow retry settings.
	// +optional
	Retries *HTTPRetry `json:"retries,omitempty"`
}

// ClusterIngressBackend describes all endpoints for a given service and port.
type ClusterIngressBackendSplit struct {
	// Specifies the backend receiving the traffic split.
	ClusterIngressBackend `json:",inline"`

	// Specifies the split percentage, a number between 0 and 100.  If
	// only one split is specified, we default to 100.
	//
	// NOTE: This differs from K8s Ingress to allow percentage split.
	Percent int `json:"percent,omitempty"`
}

// ClusterIngressBackend describes all endpoints for a given service and port.
type ClusterIngressBackend struct {
	// Specifies the namespace of the referenced service.
	//
	// NOTE: This differs from K8s Ingress to allow routing to different namespaces.
	ServiceNamespace string `json:"serviceNamespace"`

	// Specifies the name of the referenced service.
	ServiceName string `json:"serviceName"`

	// Specifies the port of the referenced service.
	ServicePort intstr.IntOrString `json:"servicePort"`
}

// HTTPRetry describes the retry policy to use when a HTTP request fails.
type HTTPRetry struct {
	// Number of retries for a given request.
	Attempts int `json:"attempts"`

	// Timeout per retry attempt for a given request. format: 1h/1m/1s/1ms. MUST BE >=1ms.
	PerTryTimeout *metav1.Duration `json:"perTryTimeout"`
}

// IngressStatus describe the current state of the ClusterIngress.
type IngressStatus struct {
	// +optional
	Conditions duckv1alpha1.Conditions `json:"conditions,omitempty"`
	// LoadBalancer contains the current status of the load-balancer.
	// +optional
	LoadBalancer *LoadBalancerStatus `json:"loadBalancer,omitempty"`
}

// LoadBalancerStatus represents the status of a load-balancer.
type LoadBalancerStatus struct {
	// Ingress is a list containing ingress points for the load-balancer.
	// Traffic intended for the service should be sent to these ingress points.
	// +optional
	Ingress []LoadBalancerIngressStatus `json:"ingress,omitempty"`
}

// LoadBalancerIngress represents the status of a load-balancer ingress point:
// traffic intended for the service should be sent to an ingress point.
type LoadBalancerIngressStatus struct {
	// IP is set for load-balancer ingress points that are IP based
	// (typically GCE or OpenStack load-balancers)
	// +optional
	IP string `json:"ip,omitempty"`

	// Domain is set for load-balancer ingress points that are DNS based
	// (typically AWS load-balancers)
	// +optional
	Domain string `json:"domain,omitempty"`

	// DomainInternal is set if there is a cluster-local DNS name to access the Ingress.
	//
	// NOTE: This differs from K8s Ingress, since we also desire to have a cluster-local
	//       DNS name to allow routing in case of not having a mesh.
	//
	// +optional
	DomainInternal string `json:"domainInternal,omitempty"`

	// MeshOnly is set if the ClusterIngress is only load-balanced through a Service mesh.
	// +optional
	MeshOnly bool `json:"meshOnly,omitempty"`
}

// ConditionType represents a ClusterIngress condition value
const (
	// ClusterIngressConditionReady is set when the clusterIngress networking setting is
	// configured and it has a load balancer address.
	ClusterIngressConditionReady = duckv1alpha1.ConditionReady

	// ClusterIngressConditionNetworkConfigured is set when the ClusterIngress's underlying
	// network programming has been configured.  This doesn't include conditions of the
	// backends, so even if this should remain true when network is configured and backends
	// are not ready.
	ClusterIngressConditionNetworkConfigured duckv1alpha1.ConditionType = "NetworkConfigured"

	// ClusterIngressConditionLoadBalancerReady is set when the ClusterIngress has
	// a ready LoadBalancer.
	ClusterIngressConditionLoadBalancerReady duckv1alpha1.ConditionType = "LoadBalancerReady"
)

var clusterIngressCondSet = duckv1alpha1.NewLivingConditionSet(
	ClusterIngressConditionNetworkConfigured,
	ClusterIngressConditionLoadBalancerReady)

var _ apis.Validatable = (*ClusterIngress)(nil)
var _ apis.Defaultable = (*ClusterIngress)(nil)

func (ci *ClusterIngress) GetGroupVersionKind() schema.GroupVersionKind {
	return SchemeGroupVersion.WithKind("ClusterIngress")
}

// IsPublic returns whether the ClusterIngress should be exposed publicly.
func (ci *ClusterIngress) IsPublic() bool {
	return ci.Spec.Visibility == "" || ci.Spec.Visibility == IngressVisibilityExternalIP
}

// GetConditions returns the Conditions array. This enables generic handling of
// conditions by implementing the duckv1alpha1.Conditions interface.
func (cis *IngressStatus) GetConditions() duckv1alpha1.Conditions {
	return cis.Conditions
}

// SetConditions sets the Conditions array. This enables generic handling of
// conditions by implementing the duckv1alpha1.Conditions interface.
func (cis *IngressStatus) SetConditions(conditions duckv1alpha1.Conditions) {
	cis.Conditions = conditions
}

func (cis *IngressStatus) GetCondition(t duckv1alpha1.ConditionType) *duckv1alpha1.Condition {
	return clusterIngressCondSet.Manage(cis).GetCondition(t)
}

func (cis *IngressStatus) InitializeConditions() {
	clusterIngressCondSet.Manage(cis).InitializeConditions()
}

func (cis *IngressStatus) MarkNetworkConfigured() {
	clusterIngressCondSet.Manage(cis).MarkTrue(ClusterIngressConditionNetworkConfigured)
}

// MarkLoadBalancerReady marks the Ingress with ClusterIngressConditionLoadBalancerReady,
// and also populate the address of the load balancer.
func (cis *IngressStatus) MarkLoadBalancerReady(lbs []LoadBalancerIngressStatus) {
	cis.LoadBalancer = &LoadBalancerStatus{
		Ingress: []LoadBalancerIngressStatus{},
	}
	for _, lb := range lbs {
		cis.LoadBalancer.Ingress = append(cis.LoadBalancer.Ingress, lb)
	}
	clusterIngressCondSet.Manage(cis).MarkTrue(ClusterIngressConditionLoadBalancerReady)
}

// IsReady looks at the conditions and if the Status has a condition
// ClusterIngressConditionReady returns true if ConditionStatus is True
func (cis *IngressStatus) IsReady() bool {
	return clusterIngressCondSet.Manage(cis).IsHappy()
}
<commit_msg>Fail with "Reason: NotOwned" when a sub-resource name is taken. (#2882)<commit_after>/*
Copyright 2018 The Knative Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
	"fmt"

	"github.com/knative/pkg/apis"
	duckv1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/intstr"
)

// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// +genclient:nonNamespaced

// ClusterIngress is a collection of rules that allow inbound connections to reach the
// endpoints defined by a backend. An ClusterIngress can be configured to give services
// externally-reachable urls, load balance traffic offer name based virtual hosting etc.
//
// This is heavily based on K8s Ingress https://godoc.org/k8s.io/api/extensions/v1beta1#Ingress
// which some highlighted modifications.
type ClusterIngress struct {
	metav1.TypeMeta `json:",inline"`
	// Standard object's metadata.
	// More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata
	// +optional
	metav1.ObjectMeta `json:"metadata,omitempty"`

	// Spec is the desired state of the ClusterIngress.
	// More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status
	// +optional
	Spec IngressSpec `json:"spec,omitempty"`

	// Status is the current state of the ClusterIngress.
	// More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status
	// +optional
	Status IngressStatus `json:"status,omitempty"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// ClusterIngressList is a collection of ClusterIngress.
type ClusterIngressList struct {
	metav1.TypeMeta `json:",inline"`
	// Standard object's metadata.
	// More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata
	// +optional
	metav1.ListMeta `json:"metadata,omitempty"`

	// Items is the list of ClusterIngress.
	Items []ClusterIngress `json:"items"`
}

// IngressSpec describes the ClusterIngress the user wishes to exist.
//
// In general this follow the same shape as K8s Ingress.  Some notable differences:
// - Backends now can have namespace:
// - Traffic can be split across multiple backends.
// - Timeout & Retry can be configured.
// - Headers can be appended.
type IngressSpec struct {
	// DeprecatedGeneration was used prior in Kubernetes versions <1.11
	// when metadata.generation was not being incremented by the api server
	//
	// This property will be dropped in future Knative releases and should
	// not be used - use metadata.generation
	//
	// Tracking issue: https://github.com/knative/serving/issues/643
	//
	// +optional
	DeprecatedGeneration int64 `json:"generation,omitempty"`

	// TLS configuration. Currently the ClusterIngress only supports a single TLS
	// port, 443. If multiple members of this list specify different hosts, they
	// will be multiplexed on the same port according to the hostname specified
	// through the SNI TLS extension, if the ingress controller fulfilling the
	// ingress supports SNI.
	// +optional
	TLS []ClusterIngressTLS `json:"tls,omitempty"`

	// A list of host rules used to configure the ClusterIngress.
	// +optional
	Rules []ClusterIngressRule `json:"rules,omitempty"`

	// Visibility setting.
	Visibility IngressVisibility `json:"visibility,omitempty"`
}

// IngressVisibility describes whether the Ingress should be exposed to
// public gateways or not.
type IngressVisibility string

const (
	// IngressVisibilityExternalIP is used to denote that the Ingress
	// should be exposed to an external IP, for example a LoadBalancer
	// Service.  This is the default value for IngressVisibility.
	IngressVisibilityExternalIP IngressVisibility = "ExternalIP"
	// IngressVisibilityClusterLocal is used to denote that the Ingress
	// should be only be exposed locally to the cluster.
	IngressVisibilityClusterLocal IngressVisibility = "ClusterLocal"
)

// ClusterIngressTLS describes the transport layer security associated with an ClusterIngress.
type ClusterIngressTLS struct {
	// Hosts are a list of hosts included in the TLS certificate. The values in
	// this list must match the name/s used in the tlsSecret. Defaults to the
	// wildcard host setting for the loadbalancer controller fulfilling this
	// ClusterIngress, if left unspecified.
	// +optional
	Hosts []string `json:"hosts,omitempty"`

	// SecretName is the name of the secret used to terminate SSL traffic.
	SecretName string `json:"secretName,omitempty"`

	// SecretNamespace is the namespace of the secret used to terminate SSL traffic.
	SecretNamespace string `json:"secretNamespace,omitempty"`

	// ServerCertificate identifies the certificate filename in the secret.
	// Defaults to `tls.cert`.
	// +optional
	ServerCertificate string `json:"serverCertificate,omitempty"`

	// PrivateKey identifies the private key filename in the secret.
	// Defaults to `tls.key`.
	// +optional
	PrivateKey string `json:"privateKey,omitempty"`
}

// ClusterIngressRule represents the rules mapping the paths under a specified host to
// the related backend services. Incoming requests are first evaluated for a host
// match, then routed to the backend associated with the matching ClusterIngressRuleValue.
type ClusterIngressRule struct {
	// Host is the fully qualified domain name of a network host, as defined
	// by RFC 3986. Note the following deviations from the "host" part of the
	// URI as defined in the RFC:
	// 1. IPs are not allowed. Currently a rule value can only apply to the
	//	  IP in the Spec of the parent ClusterIngress.
	// 2. The `:` delimiter is not respected because ports are not allowed.
	//	  Currently the port of an ClusterIngress is implicitly :80 for http and
	//	  :443 for https.
	// Both these may change in the future.
	// If the host is unspecified, the ClusterIngress routes all traffic based on the
	// specified ClusterIngressRuleValue.
	// If multiple matching Hosts were provided, the first rule will take precedent.
	// +optional
	Hosts []string `json:"hosts,omitempty"`

	// HTTP represents a rule to apply against incoming requests. If the
	// rule is satisfied, the request is routed to the specified backend.
	HTTP *HTTPClusterIngressRuleValue `json:"http,omitempty"`
}

// HTTPClusterIngressRuleValue is a list of http selectors pointing to backends.
// In the example: http://<host>/<path>?<searchpart> -> backend where
// where parts of the url correspond to RFC 3986, this resource will be used
// to match against everything after the last '/' and before the first '?'
// or '#'.
type HTTPClusterIngressRuleValue struct {
	// A collection of paths that map requests to backends.
	//
	// If they are multiple matching paths, the first match takes precendent.
	Paths []HTTPClusterIngressPath `json:"paths"`

	// TODO: Consider adding fields for ingress-type specific global
	// options usable by a loadbalancer, like http keep-alive.
}

// HTTPClusterIngressPath associates a path regex with a backend. Incoming urls matching
// the path are forwarded to the backend.
type HTTPClusterIngressPath struct {
	// Path is an extended POSIX regex as defined by IEEE Std 1003.1,
	// (i.e this follows the egrep/unix syntax, not the perl syntax)
	// matched against the path of an incoming request. Currently it can
	// contain characters disallowed from the conventional "path"
	// part of a URL as defined by RFC 3986. Paths must begin with
	// a '/'. If unspecified, the path defaults to a catch all sending
	// traffic to the backend.
	// +optional
	Path string `json:"path,omitempty"`

	// Splits defines the referenced service endpoints to which the traffic
	// will be forwarded to.
	Splits []ClusterIngressBackendSplit `json:"splits"`

	// AppendHeaders allow specifying additional HTTP headers to add
	// before forwarding a request to the destination service.
	//
	// NOTE: This differs from K8s Ingress which doesn't allow header appending.
	// +optional
	AppendHeaders map[string]string `json:"appendHeaders,omitempty"`

	// Timeout for HTTP requests.
	//
	// NOTE: This differs from K8s Ingress which doesn't allow setting timeouts.
	// +optional
	Timeout *metav1.Duration `json:"timeout,omitempty"`

	// Retry policy for HTTP requests.
	//
	// NOTE: This differs from K8s Ingress which doesn't allow retry settings.
	// +optional
	Retries *HTTPRetry `json:"retries,omitempty"`
}

// ClusterIngressBackend describes all endpoints for a given service and port.
type ClusterIngressBackendSplit struct {
	// Specifies the backend receiving the traffic split.
	ClusterIngressBackend `json:",inline"`

	// Specifies the split percentage, a number between 0 and 100.  If
	// only one split is specified, we default to 100.
	//
	// NOTE: This differs from K8s Ingress to allow percentage split.
	Percent int `json:"percent,omitempty"`
}

// ClusterIngressBackend describes all endpoints for a given service and port.
type ClusterIngressBackend struct {
	// Specifies the namespace of the referenced service.
	//
	// NOTE: This differs from K8s Ingress to allow routing to different namespaces.
	ServiceNamespace string `json:"serviceNamespace"`

	// Specifies the name of the referenced service.
	ServiceName string `json:"serviceName"`

	// Specifies the port of the referenced service.
	ServicePort intstr.IntOrString `json:"servicePort"`
}

// HTTPRetry describes the retry policy to use when a HTTP request fails.
type HTTPRetry struct {
	// Number of retries for a given request.
	Attempts int `json:"attempts"`

	// Timeout per retry attempt for a given request. format: 1h/1m/1s/1ms. MUST BE >=1ms.
	PerTryTimeout *metav1.Duration `json:"perTryTimeout"`
}

// IngressStatus describe the current state of the ClusterIngress.
type IngressStatus struct {
	// +optional
	Conditions duckv1alpha1.Conditions `json:"conditions,omitempty"`
	// LoadBalancer contains the current status of the load-balancer.
	// +optional
	LoadBalancer *LoadBalancerStatus `json:"loadBalancer,omitempty"`
}

// LoadBalancerStatus represents the status of a load-balancer.
type LoadBalancerStatus struct {
	// Ingress is a list containing ingress points for the load-balancer.
	// Traffic intended for the service should be sent to these ingress points.
	// +optional
	Ingress []LoadBalancerIngressStatus `json:"ingress,omitempty"`
}

// LoadBalancerIngress represents the status of a load-balancer ingress point:
// traffic intended for the service should be sent to an ingress point.
type LoadBalancerIngressStatus struct {
	// IP is set for load-balancer ingress points that are IP based
	// (typically GCE or OpenStack load-balancers)
	// +optional
	IP string `json:"ip,omitempty"`

	// Domain is set for load-balancer ingress points that are DNS based
	// (typically AWS load-balancers)
	// +optional
	Domain string `json:"domain,omitempty"`

	// DomainInternal is set if there is a cluster-local DNS name to access the Ingress.
	//
	// NOTE: This differs from K8s Ingress, since we also desire to have a cluster-local
	//       DNS name to allow routing in case of not having a mesh.
	//
	// +optional
	DomainInternal string `json:"domainInternal,omitempty"`

	// MeshOnly is set if the ClusterIngress is only load-balanced through a Service mesh.
	// +optional
	MeshOnly bool `json:"meshOnly,omitempty"`
}

// ConditionType represents a ClusterIngress condition value
const (
	// ClusterIngressConditionReady is set when the clusterIngress networking setting is
	// configured and it has a load balancer address.
	ClusterIngressConditionReady = duckv1alpha1.ConditionReady

	// ClusterIngressConditionNetworkConfigured is set when the ClusterIngress's underlying
	// network programming has been configured.  This doesn't include conditions of the
	// backends, so even if this should remain true when network is configured and backends
	// are not ready.
	ClusterIngressConditionNetworkConfigured duckv1alpha1.ConditionType = "NetworkConfigured"

	// ClusterIngressConditionLoadBalancerReady is set when the ClusterIngress has
	// a ready LoadBalancer.
	ClusterIngressConditionLoadBalancerReady duckv1alpha1.ConditionType = "LoadBalancerReady"
)

var clusterIngressCondSet = duckv1alpha1.NewLivingConditionSet(
	ClusterIngressConditionNetworkConfigured,
	ClusterIngressConditionLoadBalancerReady)

var _ apis.Validatable = (*ClusterIngress)(nil)
var _ apis.Defaultable = (*ClusterIngress)(nil)

func (ci *ClusterIngress) GetGroupVersionKind() schema.GroupVersionKind {
	return SchemeGroupVersion.WithKind("ClusterIngress")
}

// IsPublic returns whether the ClusterIngress should be exposed publicly.
func (ci *ClusterIngress) IsPublic() bool {
	return ci.Spec.Visibility == "" || ci.Spec.Visibility == IngressVisibilityExternalIP
}

// GetConditions returns the Conditions array. This enables generic handling of
// conditions by implementing the duckv1alpha1.Conditions interface.
func (cis *IngressStatus) GetConditions() duckv1alpha1.Conditions {
	return cis.Conditions
}

// SetConditions sets the Conditions array. This enables generic handling of
// conditions by implementing the duckv1alpha1.Conditions interface.
func (cis *IngressStatus) SetConditions(conditions duckv1alpha1.Conditions) {
	cis.Conditions = conditions
}

func (cis *IngressStatus) GetCondition(t duckv1alpha1.ConditionType) *duckv1alpha1.Condition {
	return clusterIngressCondSet.Manage(cis).GetCondition(t)
}

func (cis *IngressStatus) InitializeConditions() {
	clusterIngressCondSet.Manage(cis).InitializeConditions()
}

func (cis *IngressStatus) MarkNetworkConfigured() {
	clusterIngressCondSet.Manage(cis).MarkTrue(ClusterIngressConditionNetworkConfigured)
}

// MarkResourceNotOwned changes the "NetworkConfigured" condition to false to reflect that the
// resource of the given kind and name has already been created, and we do not own it.
func (cis *IngressStatus) MarkResourceNotOwned(kind, name string) {
	clusterIngressCondSet.Manage(cis).MarkFalse(ClusterIngressConditionNetworkConfigured, "NotOwned",
		fmt.Sprintf("There is an existing %s %q that we do not own.", kind, name))
}

// MarkLoadBalancerReady marks the Ingress with ClusterIngressConditionLoadBalancerReady,
// and also populate the address of the load balancer.
func (cis *IngressStatus) MarkLoadBalancerReady(lbs []LoadBalancerIngressStatus) {
	cis.LoadBalancer = &LoadBalancerStatus{
		Ingress: []LoadBalancerIngressStatus{},
	}
	for _, lb := range lbs {
		cis.LoadBalancer.Ingress = append(cis.LoadBalancer.Ingress, lb)
	}
	clusterIngressCondSet.Manage(cis).MarkTrue(ClusterIngressConditionLoadBalancerReady)
}

// IsReady looks at the conditions and if the Status has a condition
// ClusterIngressConditionReady returns true if ConditionStatus is True
func (cis *IngressStatus) IsReady() bool {
	return clusterIngressCondSet.Manage(cis).IsHappy()
}
<|endoftext|>


<commit_before>package upgrade

import (
	"context"
	"fmt"
	"strings"

	"github.com/rancher/rancher/pkg/controllers/user/pipeline/controller/pipelineexecution"
	"github.com/rancher/rancher/pkg/controllers/user/systemimage"
	"github.com/rancher/rancher/pkg/pipeline/utils"
	"github.com/rancher/rancher/pkg/ref"
	"github.com/rancher/rancher/pkg/systemaccount"
	rv1beta2 "github.com/rancher/types/apis/apps/v1beta2"
	"github.com/rancher/types/apis/core/v1"
	"github.com/rancher/types/config"
	"k8s.io/api/apps/v1beta2"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/labels"
)

var (
	serviceName = "pipeline"
)

type pipelineService struct {
	deployments          rv1beta2.DeploymentInterface
	namespaceLister      v1.NamespaceLister
	secrets              v1.SecretInterface
	systemAccountManager *systemaccount.Manager
}

func init() {
	systemimage.RegisterSystemService(serviceName, &pipelineService{})
}

func (l *pipelineService) Init(ctx context.Context, cluster *config.UserContext) {
	l.deployments = cluster.Apps.Deployments("")
	l.namespaceLister = cluster.Core.Namespaces("").Controller().Lister()
	l.secrets = cluster.Core.Secrets("")
	l.systemAccountManager = systemaccount.NewManager(cluster.Management)
}

func (l *pipelineService) Version() (string, error) {
	d1, d2, d3 := getDeployments()
	newJekinsVersion, err := systemimage.DefaultGetVersion(d1)
	if err != nil {
		return "", err
	}

	newRegistryVersion, err := systemimage.DefaultGetVersion(d2)
	if err != nil {
		return "", err
	}

	newMinioVersion, err := systemimage.DefaultGetVersion(d3)
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("%s-%s-%s", newJekinsVersion, newRegistryVersion, newMinioVersion), nil
}

func (l *pipelineService) Upgrade(currentVersion string) (newVersion string, err error) {
	var jekinsVersion, registryVersion, minioVersion, newJekinsVersion, newRegistryVersion, newMinioVersion string
	if currentVersion != "" {
		versions := strings.Split(currentVersion, "-")
		if len(versions) < 3 {
			return "", fmt.Errorf("invalid system service %s version %s", serviceName, currentVersion)
		}
		jekinsVersion = versions[0]
		registryVersion = versions[1]
		minioVersion = versions[2]
	}
	d1, d2, d3 := getDeployments()
	newJekinsVersion, err = systemimage.DefaultGetVersion(d1)
	if err != nil {
		return "", err
	}

	newRegistryVersion, err = systemimage.DefaultGetVersion(d2)
	if err != nil {
		return "", err
	}

	newMinioVersion, err = systemimage.DefaultGetVersion(d3)
	if err != nil {
		return "", err
	}

	set := labels.Set(map[string]string{utils.PipelineNamespaceLabel: "true"})
	pipelineNamespaces, err := l.namespaceLister.List("", set.AsSelector())
	if err != nil {
		return "", fmt.Errorf("list namespaces failed, %v", err)
	}
	for _, v := range pipelineNamespaces {
		ns := v.Name
		deployment := pipelineexecution.GetJenkinsDeployment(ns)
		if jekinsVersion != newJekinsVersion {
			if err = l.upgradeDeployment(deployment); err != nil {
				return "", err
			}
		}

		deployment = pipelineexecution.GetRegistryDeployment(ns)
		if registryVersion != newRegistryVersion {
			if err = l.upgradeDeployment(deployment); err != nil {
				return "", err
			}
		}

		deployment = pipelineexecution.GetMinioDeployment(ns)
		if minioVersion != newMinioVersion {
			if err = l.upgradeDeployment(deployment); err != nil {
				return "", err
			}
		}

		if err := l.ensureSecrets(v); err != nil {
			return "", err
		}
	}

	return fmt.Sprintf("%s-%s-%s", newJekinsVersion, newRegistryVersion, newMinioVersion), nil
}

func (l *pipelineService) ensureSecrets(namespace *corev1.Namespace) error {
	projectName := namespace.Annotations["field.cattle.io/projectId"]
	_, projectID := ref.Parse(projectName)
	ns := namespace.Name
	apikey, err := l.systemAccountManager.GetOrCreateProjectSystemToken(projectID)
	if err != nil {
		return err
	}
	secret := pipelineexecution.GetAPIKeySecret(ns, apikey)
	if _, err := l.secrets.Create(secret); err != nil && !apierrors.IsAlreadyExists(err) {
		return err
	}
	return nil
}

func (l *pipelineService) upgradeDeployment(deployment *v1beta2.Deployment) error {
	if _, err := l.deployments.Update(deployment); err != nil {
		if apierrors.IsNotFound(err) {
			return nil
		}
		return fmt.Errorf("upgrade system service %s:%s failed, %v", deployment.Namespace, deployment.Name, err)
	}
	return nil
}

func getDeployments() (d1, d2, d3 *v1beta2.Deployment) {
	d1 = pipelineexecution.GetJenkinsDeployment("")
	d2 = pipelineexecution.GetRegistryDeployment("")
	d3 = pipelineexecution.GetMinioDeployment("")
	return d1, d2, d3
}
<commit_msg>Update Pipeline component upgrade<commit_after>package upgrade

import (
	"context"
	"fmt"
	"github.com/rancher/rancher/pkg/controllers/user/pipeline/controller/pipelineexecution"
	"github.com/rancher/rancher/pkg/controllers/user/systemimage"
	"github.com/rancher/rancher/pkg/pipeline/utils"
	"github.com/rancher/rancher/pkg/ref"
	"github.com/rancher/rancher/pkg/systemaccount"
	rv1beta2 "github.com/rancher/types/apis/apps/v1beta2"
	"github.com/rancher/types/apis/core/v1"
	"github.com/rancher/types/config"
	"k8s.io/api/apps/v1beta2"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/labels"
)

var (
	serviceName = "pipeline"
)

type pipelineService struct {
	deployments          rv1beta2.DeploymentInterface
	namespaceLister      v1.NamespaceLister
	secrets              v1.SecretInterface
	systemAccountManager *systemaccount.Manager
}

func init() {
	systemimage.RegisterSystemService(serviceName, &pipelineService{})
}

func (l *pipelineService) Init(ctx context.Context, cluster *config.UserContext) {
	l.deployments = cluster.Apps.Deployments("")
	l.namespaceLister = cluster.Core.Namespaces("").Controller().Lister()
	l.secrets = cluster.Core.Secrets("")
	l.systemAccountManager = systemaccount.NewManager(cluster.Management)
}

func (l *pipelineService) Version() (string, error) {
	d1, d2, d3 := getDeployments()
	newJekinsVersion, err := systemimage.DefaultGetVersion(d1)
	if err != nil {
		return "", err
	}

	newRegistryVersion, err := systemimage.DefaultGetVersion(d2)
	if err != nil {
		return "", err
	}

	newMinioVersion, err := systemimage.DefaultGetVersion(d3)
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("%s-%s-%s", newJekinsVersion, newRegistryVersion, newMinioVersion), nil
}

func (l *pipelineService) Upgrade(currentVersion string) (newVersion string, err error) {
	var newJekinsVersion, newRegistryVersion, newMinioVersion string

	d1, d2, d3 := getDeployments()
	newJekinsVersion, err = systemimage.DefaultGetVersion(d1)
	if err != nil {
		return "", err
	}

	newRegistryVersion, err = systemimage.DefaultGetVersion(d2)
	if err != nil {
		return "", err
	}

	newMinioVersion, err = systemimage.DefaultGetVersion(d3)
	if err != nil {
		return "", err
	}

	set := labels.Set(map[string]string{utils.PipelineNamespaceLabel: "true"})
	pipelineNamespaces, err := l.namespaceLister.List("", set.AsSelector())
	if err != nil {
		return "", fmt.Errorf("list namespaces failed, %v", err)
	}
	for _, v := range pipelineNamespaces {
		if err := l.ensureSecrets(v); err != nil {
			return "", err
		}
	}

	return fmt.Sprintf("%s-%s-%s", newJekinsVersion, newRegistryVersion, newMinioVersion), nil
}

func (l *pipelineService) ensureSecrets(namespace *corev1.Namespace) error {
	projectName := namespace.Annotations["field.cattle.io/projectId"]
	_, projectID := ref.Parse(projectName)
	ns := namespace.Name
	apikey, err := l.systemAccountManager.GetOrCreateProjectSystemToken(projectID)
	if err != nil {
		return err
	}
	secret := pipelineexecution.GetAPIKeySecret(ns, apikey)
	if _, err := l.secrets.Create(secret); err != nil && !apierrors.IsAlreadyExists(err) {
		return err
	}
	return nil
}

func (l *pipelineService) upgradeDeployment(deployment *v1beta2.Deployment) error {
	if _, err := l.deployments.Update(deployment); err != nil {
		if apierrors.IsNotFound(err) {
			return nil
		}
		return fmt.Errorf("upgrade system service %s:%s failed, %v", deployment.Namespace, deployment.Name, err)
	}
	return nil
}

func getDeployments() (d1, d2, d3 *v1beta2.Deployment) {
	d1 = pipelineexecution.GetJenkinsDeployment("")
	d2 = pipelineexecution.GetRegistryDeployment("")
	d3 = pipelineexecution.GetMinioDeployment("")
	return d1, d2, d3
}
<|endoftext|>


<commit_before>package search

import (
	"bytes"
	"net/url"
	"strconv"

	"github.com/PuerkitoBio/goquery"
	"github.com/fatih/color"
)

type search struct {
	Keyword    string
	Pagenation int
	Sort       string
	Contents   []*Content
}

// Content is structure that scraping content from Qiita
type Content struct {
	ID    string
	Title string
	Desc  string
}

func New(keyword string, sort string) *search {
	return &search{
		Keyword:    keyword,
		Pagenation: 0,
		Sort:       sort,
	}
}

func (s *search) GetURL() string {
	q := url.Values{}
	q.Set("page", strconv.Itoa(s.Pagenation))
	q.Set("q", s.Keyword)
	q.Set("sort", s.Sort)
	u := url.URL{
		Scheme:   "https",
		Host:     "qiita.com",
		Path:     "search",
		RawQuery: q.Encode(),
	}

	return u.String()
}

func (s *search) NextPage() {
	s.Pagenation++
}

func (s *search) Exec(page string) ([]*Content, error) {
	doc, err := goquery.NewDocument(page)
	if err != nil {
		return nil, err
	}
	doc.Find(".searchResult").Each(s.getAttr)

	return s.Contents, nil
}

func (s *search) ContentString() string {
	var out bytes.Buffer

	for n, c := range s.Contents {
		out.WriteString(color.YellowString(strconv.Itoa(n) + " -> "))
		out.WriteString(c.Title + "\n")
		out.WriteString(color.GreenString(c.Desc) + "\n\n")
	}

	if len(s.Contents) == 10 {
		out.WriteString(color.YellowString("n -> ") + "next page\n")
	}

	return out.String()
}

func (s *search) getAttr(_ int, q *goquery.Selection) {
	itemID, _ := q.Attr("data-uuid")
	title := q.Find(".searchResult_itemTitle a").Text()
	desc := q.Find(".searchResult_snippet").Text()

	content := &Content{
		ID:    itemID,
		Title: title,
		Desc:  desc,
	}

	s.Contents = append(s.Contents, content)
}
<commit_msg>Change public struct for using control layer<commit_after>package search

import (
	"bytes"
	"net/url"
	"strconv"

	"github.com/PuerkitoBio/goquery"
	"github.com/fatih/color"
)

type Search struct {
	Keyword    string
	Pagenation int
	Sort       string
	Contents   []*Content
}

// Content is structure that scraping content from Qiita
type Content struct {
	ID    string
	Title string
	Desc  string
}

func New(keyword string, sort string) *Search {
	return &Search{
		Keyword:    keyword,
		Pagenation: 0,
		Sort:       sort,
	}
}

func (s *Search) GetURL() string {
	q := url.Values{}
	q.Set("page", strconv.Itoa(s.Pagenation))
	q.Set("q", s.Keyword)
	q.Set("sort", s.Sort)
	u := url.URL{
		Scheme:   "https",
		Host:     "qiita.com",
		Path:     "search",
		RawQuery: q.Encode(),
	}

	return u.String()
}

func (s *Search) NextPage() {
	s.Pagenation++
}

func (s *Search) Exec(page string) ([]*Content, error) {
	doc, err := goquery.NewDocument(page)
	if err != nil {
		return nil, err
	}
	doc.Find(".searchResult").Each(s.getAttr)

	return s.Contents, nil
}

func (s *Search) ContentString() string {
	var out bytes.Buffer

	for n, c := range s.Contents {
		out.WriteString(color.YellowString(strconv.Itoa(n) + " -> "))
		out.WriteString(c.Title + "\n")
		out.WriteString(color.GreenString(c.Desc) + "\n\n")
	}

	if len(s.Contents) == 10 {
		out.WriteString(color.YellowString("n -> ") + "next page\n")
	}

	return out.String()
}

func (s *Search) getAttr(_ int, q *goquery.Selection) {
	itemID, _ := q.Attr("data-uuid")
	title := q.Find(".searchResult_itemTitle a").Text()
	desc := q.Find(".searchResult_snippet").Text()

	content := &Content{
		ID:    itemID,
		Title: title,
		Desc:  desc,
	}

	s.Contents = append(s.Contents, content)
}
<|endoftext|>


<commit_before>// +build go1.9

package main

import (
	"errors"
	"net"
	"syscall"
	"unsafe"
)

type sockaddr struct {
	family uint16
	data   [14]byte
}

const SO_ORIGINAL_DST = 80

// realServerAddress returns an intercepted connection's original destination.
func realServerAddress(conn net.Conn) (net.Addr, error) {
	syscallConn, ok := conn.(syscall.Conn)
	if !ok {
		return nil, errors.New("can't get raw network connection")
	}
	rawConn, err := syscallConn.SyscallConn()
	if err != nil {
		return nil, err
	}

	var addr sockaddr
	var getsockoptErr error
	err = rawConn.Control(func(fd uintptr) {
		size := uint32(unsafe.Sizeof(addr))
		getsockoptErr = getsockopt(int(fd), syscall.SOL_IP, SO_ORIGINAL_DST, unsafe.Pointer(&addr), &size)
	})
	if err != nil {
		return nil, err
	}
	if getsockoptErr != nil {
		return nil, getsockoptErr
	}

	var ip net.IP
	switch addr.family {
	case syscall.AF_INET:
		ip = addr.data[2:6]
	default:
		return nil, errors.New("unrecognized address family")
	}

	port := int(addr.data[0])<<8 + int(addr.data[1])

	return &net.TCPAddr{IP: ip, Port: port}, nil
}
<commit_msg>Fix build tag error.<commit_after>// +build linux,go1.9

package main

import (
	"errors"
	"net"
	"syscall"
	"unsafe"
)

type sockaddr struct {
	family uint16
	data   [14]byte
}

const SO_ORIGINAL_DST = 80

// realServerAddress returns an intercepted connection's original destination.
func realServerAddress(conn net.Conn) (net.Addr, error) {
	syscallConn, ok := conn.(syscall.Conn)
	if !ok {
		return nil, errors.New("can't get raw network connection")
	}
	rawConn, err := syscallConn.SyscallConn()
	if err != nil {
		return nil, err
	}

	var addr sockaddr
	var getsockoptErr error
	err = rawConn.Control(func(fd uintptr) {
		size := uint32(unsafe.Sizeof(addr))
		getsockoptErr = getsockopt(int(fd), syscall.SOL_IP, SO_ORIGINAL_DST, unsafe.Pointer(&addr), &size)
	})
	if err != nil {
		return nil, err
	}
	if getsockoptErr != nil {
		return nil, getsockoptErr
	}

	var ip net.IP
	switch addr.family {
	case syscall.AF_INET:
		ip = addr.data[2:6]
	default:
		return nil, errors.New("unrecognized address family")
	}

	port := int(addr.data[0])<<8 + int(addr.data[1])

	return &net.TCPAddr{IP: ip, Port: port}, nil
}
<|endoftext|>


<commit_before><commit_msg>Add method to output.<commit_after><|endoftext|>


<commit_before>// SPDX-License-Identifier: ISC
// Copyright (c) 2014-2020 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package listeners_test

import (
	"crypto/tls"
	"fmt"
	"math/rand"
	"net/rpc"
	"net/rpc/jsonrpc"
	"os"
	"path"
	"path/filepath"
	"testing"

	"github.com/golang/mock/gomock"
	"github.com/stretchr/testify/assert"

	"github.com/bitmark-inc/bitmarkd/counter"
	"github.com/bitmark-inc/bitmarkd/fault"
	"github.com/bitmark-inc/bitmarkd/rpc/certificate"
	"github.com/bitmark-inc/bitmarkd/rpc/fixtures"
	"github.com/bitmark-inc/bitmarkd/rpc/listeners"
	"github.com/bitmark-inc/bitmarkd/rpc/mocks"
	"github.com/bitmark-inc/logger"
)

type Add struct{}
type AddArg struct {
	A, B int
}

func (a Add) Add(arg *AddArg, reply *int) error {
	*reply = arg.A + arg.B
	return nil
}

func TestRpcListenerServe(t *testing.T) {
	fixtures.SetupTestLogger()
	defer fixtures.TeardownTestLogger()

	port := rand.Intn(30000) + 30000
	listen := fmt.Sprintf("127.0.0.1:%d", port)
	con := listeners.RPCConfiguration{
		MaximumConnections: 5,
		Bandwidth:          10000000,
		Listen:             []string{listen},
		Certificate:        "",
		Announce:           []string{"127.0.0.1:9999"},
	}

	count := counter.Counter(0)

	s := rpc.NewServer()
	err := s.Register(Add{})
	if nil != err {
		t.Error("register with error: ", err)
		t.FailNow()
	}

	ctl := gomock.NewController(t)
	defer ctl.Finish()

	a := mocks.NewMockAnnounce(ctl)
	a.EXPECT().Set(gomock.Any(), gomock.Any()).Return(nil).Times(1)

	wd, _ := os.Getwd()
	fixturePath := path.Join(filepath.Dir(wd), "fixtures")
	tlsCertificate, fin, err := certificate.Get(
		logger.New(fixtures.LogCategory),
		"test",
		fixtures.Certificate(fixturePath),
		fixtures.Key(fixturePath),
	)
	if nil != err {
		fmt.Printf("get certificate with error: %s\n", err)
	}

	l, err := listeners.NewRPC(
		&con,
		logger.New(fixtures.LogCategory),
		&count,
		s,
		a,
		tlsCertificate,
		fin,
	)
	assert.Nil(t, err, "wrong NewRPC")

	err = l.Serve()
	assert.Nil(t, err, "wrong Serve")

	tlsConfig := tls.Config{
		InsecureSkipVerify: true,
	}

	c, err := tls.Dial("tcp", fmt.Sprintf(":%d", port), &tlsConfig)
	if nil != err {
		t.Error("dial with error: ", err)
		t.FailNow()
	}

	arg := AddArg{
		A: 2,
		B: 5,
	}
	var reply int

	client := jsonrpc.NewClient(c)
	err = client.Call("Add.Add", &arg, &reply)
	assert.Nil(t, err, "wrong client Call")
	assert.Equal(t, arg.A+arg.B, reply, "wrong result")
}

func TestRpcListenerServeWhenMaxConnectionCountTooSmall(t *testing.T) {
	fixtures.SetupTestLogger()
	defer fixtures.TeardownTestLogger()

	port := rand.Intn(30000) + 30000
	listen := fmt.Sprintf("127.0.0.1:%d", port)
	con := listeners.RPCConfiguration{
		MaximumConnections: 0,
		Bandwidth:          10000000,
		Listen:             []string{listen},
		Certificate:        "",
		Announce:           []string{"127.0.0.1:9999"},
	}

	count := counter.Counter(0)

	s := rpc.NewServer()

	ctl := gomock.NewController(t)
	defer ctl.Finish()

	a := mocks.NewMockAnnounce(ctl)

	_, err := listeners.NewRPC(
		&con,
		logger.New(fixtures.LogCategory),
		&count,
		s,
		a,
		&tls.Config{},
		[32]byte{},
	)
	assert.Equal(t, fault.MissingParameters, err, "wrong error")
}

func TestRpcListenerServeWhenBandwidthTooSmall(t *testing.T) {
	fixtures.SetupTestLogger()
	defer fixtures.TeardownTestLogger()

	port := rand.Intn(30000) + 30000
	listen := fmt.Sprintf("127.0.0.1:%d", port)
	con := listeners.RPCConfiguration{
		MaximumConnections: 1,
		Bandwidth:          100,
		Listen:             []string{listen},
		Certificate:        "",
		Announce:           []string{"127.0.0.1:9999"},
	}

	count := counter.Counter(0)

	s := rpc.NewServer()

	ctl := gomock.NewController(t)
	defer ctl.Finish()

	a := mocks.NewMockAnnounce(ctl)

	_, err := listeners.NewRPC(
		&con,
		logger.New(fixtures.LogCategory),
		&count,
		s,
		a,
		&tls.Config{},
		[32]byte{},
	)
	assert.Equal(t, fault.MissingParameters, err, "wrong error")
}

func TestRpcListenerServeWhenEmptyListen(t *testing.T) {
	fixtures.SetupTestLogger()
	defer fixtures.TeardownTestLogger()

	con := listeners.RPCConfiguration{
		MaximumConnections: 1,
		Bandwidth:          10000000,
		Listen:             []string{},
		Certificate:        "",
		Announce:           []string{"127.0.0.1:9999"},
	}

	count := counter.Counter(0)

	s := rpc.NewServer()

	ctl := gomock.NewController(t)
	defer ctl.Finish()

	a := mocks.NewMockAnnounce(ctl)

	_, err := listeners.NewRPC(
		&con,
		logger.New(fixtures.LogCategory),
		&count,
		s,
		a,
		&tls.Config{},
		[32]byte{},
	)
	assert.Equal(t, fault.MissingParameters, err, "wrong error")
}

func TestRpcListenerServeWhenWrongAnnounce(t *testing.T) {
	fixtures.SetupTestLogger()
	defer fixtures.TeardownTestLogger()

	port := rand.Intn(30000) + 30000
	listen := fmt.Sprintf("127.0.0.1:%d", port)
	con := listeners.RPCConfiguration{
		MaximumConnections: 1,
		Bandwidth:          10000000,
		Listen:             []string{listen},
		Certificate:        "",
		Announce:           []string{"", "1"},
	}

	count := counter.Counter(0)

	s := rpc.NewServer()

	ctl := gomock.NewController(t)
	defer ctl.Finish()

	a := mocks.NewMockAnnounce(ctl)

	_, err := listeners.NewRPC(
		&con,
		logger.New(fixtures.LogCategory),
		&count,
		s,
		a,
		&tls.Config{},
		[32]byte{},
	)
	assert.NotNil(t, err, "wrong error")
}

func TestRpcListenerServeWhenErrorAnnounceSet(t *testing.T) {
	fixtures.SetupTestLogger()
	defer fixtures.TeardownTestLogger()

	port := rand.Intn(30000) + 30000
	listen := fmt.Sprintf("127.0.0.1:%d", port)
	con := listeners.RPCConfiguration{
		MaximumConnections: 5,
		Bandwidth:          10000000,
		Listen:             []string{listen},
		Certificate:        "",
		Announce:           []string{"127.0.0.1:9999"},
	}

	count := counter.Counter(0)

	s := rpc.NewServer()

	ctl := gomock.NewController(t)
	defer ctl.Finish()

	a := mocks.NewMockAnnounce(ctl)
	a.EXPECT().Set(gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake error")).Times(1)

	_, err := listeners.NewRPC(
		&con,
		logger.New(fixtures.LogCategory),
		&count,
		s,
		a,
		&tls.Config{},
		[32]byte{},
	)
	assert.NotNil(t, err, "wrong error")
	assert.Equal(t, "fake error", err.Error(), "wrong error message")
}

func TestRpcListenerServeWhenErrorListen(t *testing.T) {
	fixtures.SetupTestLogger()
	defer fixtures.TeardownTestLogger()

	con := listeners.RPCConfiguration{
		MaximumConnections: 5,
		Bandwidth:          10000000,
		Listen:             []string{"1"},
		Certificate:        "",
		Announce:           []string{"127.0.0.1:9999"},
	}

	count := counter.Counter(0)

	s := rpc.NewServer()

	ctl := gomock.NewController(t)
	defer ctl.Finish()

	a := mocks.NewMockAnnounce(ctl)
	a.EXPECT().Set(gomock.Any(), gomock.Any()).Return(nil).Times(1)

	_, err := listeners.NewRPC(
		&con,
		logger.New(fixtures.LogCategory),
		&count,
		s,
		a,
		&tls.Config{},
		[32]byte{},
	)
	assert.NotNil(t, err, "wrong error")
	assert.Equal(t, fault.InvalidIpAddress, err, "wrong error message")
}

func TestRpcListenerServeWhenListenAll(t *testing.T) {
	fixtures.SetupTestLogger()
	defer fixtures.TeardownTestLogger()

	con := listeners.RPCConfiguration{
		MaximumConnections: 5,
		Bandwidth:          10000000,
		Listen:             []string{"*:1234"},
		Certificate:        "",
		Announce:           []string{"127.0.0.1:9999"},
	}

	count := counter.Counter(0)

	s := rpc.NewServer()

	ctl := gomock.NewController(t)
	defer ctl.Finish()

	a := mocks.NewMockAnnounce(ctl)
	a.EXPECT().Set(gomock.Any(), gomock.Any()).Return(nil).Times(1)

	_, err := listeners.NewRPC(
		&con,
		logger.New(fixtures.LogCategory),
		&count,
		s,
		a,
		&tls.Config{},
		[32]byte{},
	)
	assert.Nil(t, err, "wrong NewRPC")
}

func TestRpcListenerServeWhenListenIPv6(t *testing.T) {
	fixtures.SetupTestLogger()
	defer fixtures.TeardownTestLogger()

	con := listeners.RPCConfiguration{
		MaximumConnections: 5,
		Bandwidth:          10000000,
		Listen:             []string{"[1:2:3:4:5:6:7:8]::1"},
		Certificate:        "",
		Announce:           []string{"127.0.0.1:9999"},
	}

	count := counter.Counter(0)

	s := rpc.NewServer()

	ctl := gomock.NewController(t)
	defer ctl.Finish()

	a := mocks.NewMockAnnounce(ctl)
	a.EXPECT().Set(gomock.Any(), gomock.Any()).Return(nil).Times(1)

	_, err := listeners.NewRPC(
		&con,
		logger.New(fixtures.LogCategory),
		&count,
		s,
		a,
		&tls.Config{},
		[32]byte{},
	)
	assert.Nil(t, err, "wrong NewRPC")
}

func TestRpcListenerServeWhenInvalidTLSConfig(t *testing.T) {
	fixtures.SetupTestLogger()
	defer fixtures.TeardownTestLogger()

	port := rand.Intn(30000) + 30000
	listen := fmt.Sprintf("127.0.0.1:%d", port)
	con := listeners.RPCConfiguration{
		MaximumConnections: 5,
		Bandwidth:          10000000,
		Listen:             []string{listen},
		Certificate:        "",
		Announce:           []string{"127.0.0.1:9999"},
	}

	count := counter.Counter(0)

	s := rpc.NewServer()

	ctl := gomock.NewController(t)
	defer ctl.Finish()

	a := mocks.NewMockAnnounce(ctl)
	a.EXPECT().Set(gomock.Any(), gomock.Any()).Return(nil).Times(1)

	l, err := listeners.NewRPC(
		&con,
		logger.New(fixtures.LogCategory),
		&count,
		s,
		a,
		&tls.Config{},
		[32]byte{},
	)
	assert.Nil(t, err, "wrong NewRPC")

	err = l.Serve()
	assert.NotNil(t, err, "wrong Serve")
	assert.Contains(t, err.Error(), "tls", "wrong error message")
}
<commit_msg>[rpc] test must connect to 127.0.0.1<commit_after>// SPDX-License-Identifier: ISC
// Copyright (c) 2014-2020 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package listeners_test

import (
	"crypto/tls"
	"fmt"
	"math/rand"
	"net/rpc"
	"net/rpc/jsonrpc"
	"os"
	"path"
	"path/filepath"
	"testing"

	"github.com/golang/mock/gomock"
	"github.com/stretchr/testify/assert"

	"github.com/bitmark-inc/bitmarkd/counter"
	"github.com/bitmark-inc/bitmarkd/fault"
	"github.com/bitmark-inc/bitmarkd/rpc/certificate"
	"github.com/bitmark-inc/bitmarkd/rpc/fixtures"
	"github.com/bitmark-inc/bitmarkd/rpc/listeners"
	"github.com/bitmark-inc/bitmarkd/rpc/mocks"
	"github.com/bitmark-inc/logger"
)

type Add struct{}
type AddArg struct {
	A, B int
}

func (a Add) Add(arg *AddArg, reply *int) error {
	*reply = arg.A + arg.B
	return nil
}

func TestRpcListenerServe(t *testing.T) {
	fixtures.SetupTestLogger()
	defer fixtures.TeardownTestLogger()

	port := rand.Intn(30000) + 30000
	listen := fmt.Sprintf("127.0.0.1:%d", port)
	con := listeners.RPCConfiguration{
		MaximumConnections: 5,
		Bandwidth:          10000000,
		Listen:             []string{listen},
		Certificate:        "",
		Announce:           []string{"127.0.0.1:9999"},
	}

	count := counter.Counter(0)

	s := rpc.NewServer()
	err := s.Register(Add{})
	if nil != err {
		t.Error("register with error: ", err)
		t.FailNow()
	}

	ctl := gomock.NewController(t)
	defer ctl.Finish()

	a := mocks.NewMockAnnounce(ctl)
	a.EXPECT().Set(gomock.Any(), gomock.Any()).Return(nil).Times(1)

	wd, _ := os.Getwd()
	fixturePath := path.Join(filepath.Dir(wd), "fixtures")
	tlsCertificate, fin, err := certificate.Get(
		logger.New(fixtures.LogCategory),
		"test",
		fixtures.Certificate(fixturePath),
		fixtures.Key(fixturePath),
	)
	if nil != err {
		fmt.Printf("get certificate with error: %s\n", err)
	}

	l, err := listeners.NewRPC(
		&con,
		logger.New(fixtures.LogCategory),
		&count,
		s,
		a,
		tlsCertificate,
		fin,
	)
	assert.Nil(t, err, "wrong NewRPC")

	err = l.Serve()
	assert.Nil(t, err, "wrong Serve")

	tlsConfig := tls.Config{
		InsecureSkipVerify: true,
	}

	c, err := tls.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", port), &tlsConfig)
	if nil != err {
		t.Error("dial with error: ", err)
		t.FailNow()
	}

	arg := AddArg{
		A: 2,
		B: 5,
	}
	var reply int

	client := jsonrpc.NewClient(c)
	err = client.Call("Add.Add", &arg, &reply)
	assert.Nil(t, err, "wrong client Call")
	assert.Equal(t, arg.A+arg.B, reply, "wrong result")
}

func TestRpcListenerServeWhenMaxConnectionCountTooSmall(t *testing.T) {
	fixtures.SetupTestLogger()
	defer fixtures.TeardownTestLogger()

	port := rand.Intn(30000) + 30000
	listen := fmt.Sprintf("127.0.0.1:%d", port)
	con := listeners.RPCConfiguration{
		MaximumConnections: 0,
		Bandwidth:          10000000,
		Listen:             []string{listen},
		Certificate:        "",
		Announce:           []string{"127.0.0.1:9999"},
	}

	count := counter.Counter(0)

	s := rpc.NewServer()

	ctl := gomock.NewController(t)
	defer ctl.Finish()

	a := mocks.NewMockAnnounce(ctl)

	_, err := listeners.NewRPC(
		&con,
		logger.New(fixtures.LogCategory),
		&count,
		s,
		a,
		&tls.Config{},
		[32]byte{},
	)
	assert.Equal(t, fault.MissingParameters, err, "wrong error")
}

func TestRpcListenerServeWhenBandwidthTooSmall(t *testing.T) {
	fixtures.SetupTestLogger()
	defer fixtures.TeardownTestLogger()

	port := rand.Intn(30000) + 30000
	listen := fmt.Sprintf("127.0.0.1:%d", port)
	con := listeners.RPCConfiguration{
		MaximumConnections: 1,
		Bandwidth:          100,
		Listen:             []string{listen},
		Certificate:        "",
		Announce:           []string{"127.0.0.1:9999"},
	}

	count := counter.Counter(0)

	s := rpc.NewServer()

	ctl := gomock.NewController(t)
	defer ctl.Finish()

	a := mocks.NewMockAnnounce(ctl)

	_, err := listeners.NewRPC(
		&con,
		logger.New(fixtures.LogCategory),
		&count,
		s,
		a,
		&tls.Config{},
		[32]byte{},
	)
	assert.Equal(t, fault.MissingParameters, err, "wrong error")
}

func TestRpcListenerServeWhenEmptyListen(t *testing.T) {
	fixtures.SetupTestLogger()
	defer fixtures.TeardownTestLogger()

	con := listeners.RPCConfiguration{
		MaximumConnections: 1,
		Bandwidth:          10000000,
		Listen:             []string{},
		Certificate:        "",
		Announce:           []string{"127.0.0.1:9999"},
	}

	count := counter.Counter(0)

	s := rpc.NewServer()

	ctl := gomock.NewController(t)
	defer ctl.Finish()

	a := mocks.NewMockAnnounce(ctl)

	_, err := listeners.NewRPC(
		&con,
		logger.New(fixtures.LogCategory),
		&count,
		s,
		a,
		&tls.Config{},
		[32]byte{},
	)
	assert.Equal(t, fault.MissingParameters, err, "wrong error")
}

func TestRpcListenerServeWhenWrongAnnounce(t *testing.T) {
	fixtures.SetupTestLogger()
	defer fixtures.TeardownTestLogger()

	port := rand.Intn(30000) + 30000
	listen := fmt.Sprintf("127.0.0.1:%d", port)
	con := listeners.RPCConfiguration{
		MaximumConnections: 1,
		Bandwidth:          10000000,
		Listen:             []string{listen},
		Certificate:        "",
		Announce:           []string{"", "1"},
	}

	count := counter.Counter(0)

	s := rpc.NewServer()

	ctl := gomock.NewController(t)
	defer ctl.Finish()

	a := mocks.NewMockAnnounce(ctl)

	_, err := listeners.NewRPC(
		&con,
		logger.New(fixtures.LogCategory),
		&count,
		s,
		a,
		&tls.Config{},
		[32]byte{},
	)
	assert.NotNil(t, err, "wrong error")
}

func TestRpcListenerServeWhenErrorAnnounceSet(t *testing.T) {
	fixtures.SetupTestLogger()
	defer fixtures.TeardownTestLogger()

	port := rand.Intn(30000) + 30000
	listen := fmt.Sprintf("127.0.0.1:%d", port)
	con := listeners.RPCConfiguration{
		MaximumConnections: 5,
		Bandwidth:          10000000,
		Listen:             []string{listen},
		Certificate:        "",
		Announce:           []string{"127.0.0.1:9999"},
	}

	count := counter.Counter(0)

	s := rpc.NewServer()

	ctl := gomock.NewController(t)
	defer ctl.Finish()

	a := mocks.NewMockAnnounce(ctl)
	a.EXPECT().Set(gomock.Any(), gomock.Any()).Return(fmt.Errorf("fake error")).Times(1)

	_, err := listeners.NewRPC(
		&con,
		logger.New(fixtures.LogCategory),
		&count,
		s,
		a,
		&tls.Config{},
		[32]byte{},
	)
	assert.NotNil(t, err, "wrong error")
	assert.Equal(t, "fake error", err.Error(), "wrong error message")
}

func TestRpcListenerServeWhenErrorListen(t *testing.T) {
	fixtures.SetupTestLogger()
	defer fixtures.TeardownTestLogger()

	con := listeners.RPCConfiguration{
		MaximumConnections: 5,
		Bandwidth:          10000000,
		Listen:             []string{"1"},
		Certificate:        "",
		Announce:           []string{"127.0.0.1:9999"},
	}

	count := counter.Counter(0)

	s := rpc.NewServer()

	ctl := gomock.NewController(t)
	defer ctl.Finish()

	a := mocks.NewMockAnnounce(ctl)
	a.EXPECT().Set(gomock.Any(), gomock.Any()).Return(nil).Times(1)

	_, err := listeners.NewRPC(
		&con,
		logger.New(fixtures.LogCategory),
		&count,
		s,
		a,
		&tls.Config{},
		[32]byte{},
	)
	assert.NotNil(t, err, "wrong error")
	assert.Equal(t, fault.InvalidIpAddress, err, "wrong error message")
}

func TestRpcListenerServeWhenListenAll(t *testing.T) {
	fixtures.SetupTestLogger()
	defer fixtures.TeardownTestLogger()

	con := listeners.RPCConfiguration{
		MaximumConnections: 5,
		Bandwidth:          10000000,
		Listen:             []string{"*:1234"},
		Certificate:        "",
		Announce:           []string{"127.0.0.1:9999"},
	}

	count := counter.Counter(0)

	s := rpc.NewServer()

	ctl := gomock.NewController(t)
	defer ctl.Finish()

	a := mocks.NewMockAnnounce(ctl)
	a.EXPECT().Set(gomock.Any(), gomock.Any()).Return(nil).Times(1)

	_, err := listeners.NewRPC(
		&con,
		logger.New(fixtures.LogCategory),
		&count,
		s,
		a,
		&tls.Config{},
		[32]byte{},
	)
	assert.Nil(t, err, "wrong NewRPC")
}

func TestRpcListenerServeWhenListenIPv6(t *testing.T) {
	fixtures.SetupTestLogger()
	defer fixtures.TeardownTestLogger()

	con := listeners.RPCConfiguration{
		MaximumConnections: 5,
		Bandwidth:          10000000,
		Listen:             []string{"[1:2:3:4:5:6:7:8]::1"},
		Certificate:        "",
		Announce:           []string{"127.0.0.1:9999"},
	}

	count := counter.Counter(0)

	s := rpc.NewServer()

	ctl := gomock.NewController(t)
	defer ctl.Finish()

	a := mocks.NewMockAnnounce(ctl)
	a.EXPECT().Set(gomock.Any(), gomock.Any()).Return(nil).Times(1)

	_, err := listeners.NewRPC(
		&con,
		logger.New(fixtures.LogCategory),
		&count,
		s,
		a,
		&tls.Config{},
		[32]byte{},
	)
	assert.Nil(t, err, "wrong NewRPC")
}

func TestRpcListenerServeWhenInvalidTLSConfig(t *testing.T) {
	fixtures.SetupTestLogger()
	defer fixtures.TeardownTestLogger()

	port := rand.Intn(30000) + 30000
	listen := fmt.Sprintf("127.0.0.1:%d", port)
	con := listeners.RPCConfiguration{
		MaximumConnections: 5,
		Bandwidth:          10000000,
		Listen:             []string{listen},
		Certificate:        "",
		Announce:           []string{"127.0.0.1:9999"},
	}

	count := counter.Counter(0)

	s := rpc.NewServer()

	ctl := gomock.NewController(t)
	defer ctl.Finish()

	a := mocks.NewMockAnnounce(ctl)
	a.EXPECT().Set(gomock.Any(), gomock.Any()).Return(nil).Times(1)

	l, err := listeners.NewRPC(
		&con,
		logger.New(fixtures.LogCategory),
		&count,
		s,
		a,
		&tls.Config{},
		[32]byte{},
	)
	assert.Nil(t, err, "wrong NewRPC")

	err = l.Serve()
	assert.NotNil(t, err, "wrong Serve")
	assert.Contains(t, err.Error(), "tls", "wrong error message")
}
<|endoftext|>