<commit_before>/*
 * Copyright (c) 2020 the Octant contributors. All Rights Reserved.
 * SPDX-License-Identifier: Apache-2.0
 */

package api

import (
	"context"
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"os"
	"sync/atomic"
	"time"

	"github.com/vmware-tanzu/octant/pkg/event"

	"github.com/spf13/afero"
	"k8s.io/client-go/tools/clientcmd"

	ocontext "github.com/vmware-tanzu/octant/internal/context"
	"github.com/vmware-tanzu/octant/internal/octant"
	"github.com/vmware-tanzu/octant/pkg/action"
)

const (
	UploadKubeConfig = "action.octant.dev/uploadKubeConfig"
	CheckLoading     = "action.octant.dev/loading"
)

type LoadingManager struct {
	client         atomic.Value
	kubeConfigPath chan string
	ctx            context.Context
}

var _ StateManager = (*LoadingManager)(nil)

func NewLoadingManager() *LoadingManager {
	lm := &LoadingManager{}

	return lm
}

func (l *LoadingManager) Handlers() []octant.ClientRequestHandler {
	return []octant.ClientRequestHandler{
		{
			RequestType: UploadKubeConfig,
			Handler:     l.UploadKubeConfig,
		},
		{
			RequestType: CheckLoading,
			Handler:     l.CheckLoading,
		},
	}
}

func (l *LoadingManager) Start(ctx context.Context, state octant.State, client OctantClient) {
	l.client.Store(client)
	l.ctx = ctx
	l.kubeConfigPath = ocontext.KubeConfigChFrom(ctx)

	fs := afero.NewOsFs()

	// Watch for config and reset router if found
	// See https://github.com/gorilla/mux/issues/82#issuecomment-121411186
	go l.WatchConfig(l.kubeConfigPath, client, fs)
}

func (l *LoadingManager) CheckLoading(state octant.State, payload action.Payload) error {
	loading, err := payload.Bool("loading")
	if err != nil {
		return fmt.Errorf("getting loading from payload: %w", err)
	}

	if loading {
		client := l.client.Load().(OctantClient)
		client.Send(event.Event{
			Type: event.EventTypeLoading,
		})
	}

	return nil
}

func (l *LoadingManager) UploadKubeConfig(state octant.State, payload action.Payload) error {
	kubeConfig64, err := payload.String("kubeConfig")
	if err != nil {
		return fmt.Errorf("getting kubeConfig from payload: %w", err)
	}

	kubeConfig, err := base64.StdEncoding.DecodeString(kubeConfig64)
	if err != nil {
		return fmt.Errorf("decode base64 kubeconfig: %w", err)
	}

	// Check if kube config can be converted to config object
	// TODO: Show error elsewhere instead of notifier
	if _, err := clientcmd.Load(kubeConfig); err != nil {
		message := fmt.Sprintf("Error parsing KubeConfig: %v", err)
		alert := action.CreateAlert(action.AlertTypeError, message, action.DefaultAlertExpiration)
		state.SendAlert(alert)
		return err
	}

	tempFile, err := ioutil.TempFile(os.TempDir(), "kubeconfig")
	if err != nil {
		return err
	}

	if _, err := tempFile.Write(kubeConfig); err != nil {
		return err
	}

	if err := tempFile.Close(); err != nil {
		return err
	}

	client := l.client.Load().(OctantClient)
	client.Send(event.Event{
		Type: event.EventTypeRefresh,
	})

	l.kubeConfigPath <- tempFile.Name()
	return nil
}

func (l *LoadingManager) WatchConfig(path chan string, client OctantClient, fs afero.Fs) {
	kubeconfig := clientcmd.NewDefaultClientConfigLoadingRules().GetDefaultFilename()

	for {
		time.Sleep(1 * time.Second)
		exists, err := afero.Exists(fs, kubeconfig)
		if err != nil {
			return
		}
		if exists {
			path <- kubeconfig
			client.Send(event.Event{
				Type: event.EventTypeRefresh,
			})
			return
		}
	}
}
<commit_msg>use a consistent and known temp sub folder in the loading handler<commit_after>/*
 * Copyright (c) 2020 the Octant contributors. All Rights Reserved.
 * SPDX-License-Identifier: Apache-2.0
 */

package api

import (
	"context"
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"sync/atomic"
	"time"

	"github.com/vmware-tanzu/octant/pkg/event"

	"github.com/spf13/afero"
	"k8s.io/client-go/tools/clientcmd"

	ocontext "github.com/vmware-tanzu/octant/internal/context"
	"github.com/vmware-tanzu/octant/internal/octant"
	"github.com/vmware-tanzu/octant/pkg/action"
)

const (
	UploadKubeConfig = "action.octant.dev/uploadKubeConfig"
	CheckLoading     = "action.octant.dev/loading"
)

type LoadingManager struct {
	client         atomic.Value
	kubeConfigPath chan string
	ctx            context.Context
}

var _ StateManager = (*LoadingManager)(nil)

func NewLoadingManager() *LoadingManager {
	lm := &LoadingManager{}

	return lm
}

func (l *LoadingManager) Handlers() []octant.ClientRequestHandler {
	return []octant.ClientRequestHandler{
		{
			RequestType: UploadKubeConfig,
			Handler:     l.UploadKubeConfig,
		},
		{
			RequestType: CheckLoading,
			Handler:     l.CheckLoading,
		},
	}
}

func (l *LoadingManager) Start(ctx context.Context, state octant.State, client OctantClient) {
	l.client.Store(client)
	l.ctx = ctx
	l.kubeConfigPath = ocontext.KubeConfigChFrom(ctx)

	fs := afero.NewOsFs()

	// Watch for config and reset router if found
	// See https://github.com/gorilla/mux/issues/82#issuecomment-121411186
	go l.WatchConfig(l.kubeConfigPath, client, fs)
}

func (l *LoadingManager) CheckLoading(state octant.State, payload action.Payload) error {
	loading, err := payload.Bool("loading")
	if err != nil {
		return fmt.Errorf("getting loading from payload: %w", err)
	}

	if loading {
		client := l.client.Load().(OctantClient)
		client.Send(event.Event{
			Type: event.EventTypeLoading,
		})
	}

	return nil
}

func (l *LoadingManager) UploadKubeConfig(state octant.State, payload action.Payload) error {
	kubeConfig64, err := payload.String("kubeConfig")
	if err != nil {
		return fmt.Errorf("getting kubeConfig from payload: %w", err)
	}

	kubeConfig, err := base64.StdEncoding.DecodeString(kubeConfig64)
	if err != nil {
		return fmt.Errorf("decode base64 kubeconfig: %w", err)
	}

	// Check if kube config can be converted to config object
	// TODO: Show error elsewhere instead of notifier
	if _, err := clientcmd.Load(kubeConfig); err != nil {
		message := fmt.Sprintf("Error parsing KubeConfig: %v", err)
		alert := action.CreateAlert(action.AlertTypeError, message, action.DefaultAlertExpiration)
		state.SendAlert(alert)
		return err
	}

	tempDir := path.Join(os.TempDir(), "octant")
	if err := os.MkdirAll(tempDir, 0755); err != nil {
		return err
	}

	tempFile, err := ioutil.TempFile(tempDir, "kubeconfig")
	if err != nil {
		return err
	}

	if _, err := tempFile.Write(kubeConfig); err != nil {
		return err
	}

	if err := tempFile.Close(); err != nil {
		return err
	}

	client := l.client.Load().(OctantClient)
	client.Send(event.Event{
		Type: event.EventTypeRefresh,
	})

	l.kubeConfigPath <- tempFile.Name()
	return nil
}

func (l *LoadingManager) WatchConfig(path chan string, client OctantClient, fs afero.Fs) {
	kubeconfig := clientcmd.NewDefaultClientConfigLoadingRules().GetDefaultFilename()

	for {
		time.Sleep(1 * time.Second)
		exists, err := afero.Exists(fs, kubeconfig)
		if err != nil {
			return
		}
		if exists {
			path <- kubeconfig
			client.Send(event.Event{
				Type: event.EventTypeRefresh,
			})
			return
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"math/big"
	"os"
	"path/filepath"
	"time"

	"github.com/ethereum/go-ethereum/accounts/keystore"
	ethcommon "github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/console"
	"github.com/ethereum/go-ethereum/ethclient"
	"github.com/ethereum/go-ethereum/rpc"
	"github.com/livepeer/go-livepeer/eth"

	"github.com/golang/glog"
)

const (
	gethMiningAccount = "87da6a8c6e9eff15d703fc2773e32f6af8dbe301"
	controllerAddr    = "93ad00a63b14492386df9f1cc123d785705bdf99"
	clientIdentifier  = "geth" // Client identifier to advertise over the network
	passphrase        = ""
	serviceURI        = "https://127.0.0.1:8936"
)

var (
	ethTxTimeout = 600 * time.Second
	endpoint     = "ws://localhost:8546/"
)

func main() {
	flag.Set("logtostderr", "true")
	baseDataDir := flag.String("datadir", ".lpdev2", "default data directory")
	endpointAddr := flag.String("endpoint", "", "Geth endpoint to connect to")

	flag.Parse()
	if *endpointAddr != "" {
		endpoint = *endpointAddr
	}
	args := flag.Args()
	goodToGo := false
	isBroadcaster := true
	if len(args) > 1 && args[0] == "setup" {
		switch args[1] {
		case "broadcaster":
			goodToGo = true
		case "transcoder":
			isBroadcaster = false
			goodToGo = true
		}
	}
	if !goodToGo {
		fmt.Println(`
    Usage: go run cmd/devtool/devtool.go setup broadcaster|transcoder
        It will create initilize eth account (on private testnet) to be used for broadcaster or transcoder
        and will create shell script (run_broadcaster_ETHACC.sh or run_transcoder_ETHACC.sh) to run it.`)
		return
	}

	t := getNodeType(isBroadcaster)

	tmp, err := ioutil.TempDir("", "livepeer")
	if err != nil {
		glog.Fatalf("Can't create temporary directory: %v", err)
	}
	defer os.RemoveAll(tmp)

	tempKeystoreDir := filepath.Join(tmp, "keystore")
	acc := createKey(tempKeystoreDir)
	glog.Infof("Using account %s", acc)
	dataDir := filepath.Join(*baseDataDir, t+"_"+acc)
	dataDirToCreate := filepath.Join(dataDir, "mainnet")
	err = os.MkdirAll(dataDirToCreate, 0755)
	if err != nil {
		glog.Fatalf("Can't create directory %v", err)
	}

	keystoreDir := filepath.Join(dataDirToCreate, "keystore")
	err = os.Rename(tempKeystoreDir, keystoreDir)
	if err != nil {
		glog.Fatal(err)
	}
	remoteConsole(acc)
	ethSetup(acc, keystoreDir, isBroadcaster)
	createRunScript(acc, dataDir, isBroadcaster)
	glog.Info("Finished")
}

func getNodeType(isBroadcaster bool) string {
	t := "broadcaster"
	if !isBroadcaster {
		t = "transcoder"
	}
	return t
}

func ethSetup(ethAcctAddr, keystoreDir string, isBroadcaster bool) {
	time.Sleep(3 * time.Second)
	//Set up eth client
	backend, err := ethclient.Dial(endpoint)
	if err != nil {
		glog.Errorf("Failed to connect to Ethereum client: %v", err)
		return
	}

	client, err := eth.NewClient(ethcommon.HexToAddress(ethAcctAddr), keystoreDir, backend,
		ethcommon.HexToAddress(controllerAddr), ethTxTimeout)
	if err != nil {
		glog.Errorf("Failed to create client: %v", err)
		return
	}

	var bigGasPrice *big.Int = big.NewInt(int64(200))
	// var bigGasPrice *big.Int = big.NewInt(int64(00000))

	err = client.Setup(passphrase, 1000000, bigGasPrice)
	if err != nil {
		glog.Errorf("Failed to setup client: %v", err)
		return
	}
	glog.Infof("Requesting tokens from faucet")

	tx, err := client.Request()
	if err != nil {
		glog.Errorf("Error requesting tokens from faucet: %v", err)
		return
	}

	err = client.CheckTx(tx)
	if err != nil {
		glog.Errorf("Error requesting tokens from faucet: %v", err)
		return
	}
	glog.Info("Done requesting tokens.")
	time.Sleep(4 * time.Second)

	var depositAmount *big.Int = big.NewInt(int64(5000))

	glog.Infof("Depositing: %v", depositAmount)

	tx, err = client.Deposit(depositAmount)
	if err != nil {
		glog.Error(err)
		return
	}
	err = client.CheckTx(tx)
	if err != nil {
		glog.Error(err)
		return
	}
	glog.Info("Done depositing")
	if !isBroadcaster {
		// XXX TODO curl -X "POST" http://localhost:$transcoderCliPort/initializeRound
		time.Sleep(3 * time.Second)
		for {
			currentRound, err := client.CurrentRound()
			if err != nil {
				glog.Errorf("Error getting current round: %v", err)
				return
			}
			if currentRound.Int64() > 1 {
				break
			}
			// first round is initialized and locked, need to wait
			glog.Info("Waiting will first round ended.")
			time.Sleep(4 * time.Second)
		}
		tx, err := client.InitializeRound()
		// ErrRoundInitialized
		if err != nil {
			if err.Error() != "ErrRoundInitialized" {
				glog.Errorf("Error initializing round: %v", err)
				return
			}
		} else {
			err = client.CheckTx(tx)
			if err != nil {
				glog.Errorf("Error initializng round: %v", err)
				return
			}
		}
		glog.Info("Done initializing round.")
		glog.Info("Activating transcoder")
		// curl -d "blockRewardCut=10&feeShare=5&pricePerSegment=1&amount=500" --data-urlencode "serviceURI=https://$transcoderServiceAddr" \
		//   -H "Content-Type: application/x-www-form-urlencoded" \
		//   -X "POST" http://localhost:$transcoderCliPort/activateTranscoder\
		var amount *big.Int = big.NewInt(int64(500))
		glog.Infof("Bonding %v to %s", amount, ethAcctAddr)

		tx, err = client.Bond(amount, ethcommon.HexToAddress(ethAcctAddr))
		if err != nil {
			glog.Error(err)
			return
		}

		err = client.CheckTx(tx)
		if err != nil {
			glog.Error(err)
			return
		}
		glog.Infof("Registering transcoder %v", ethAcctAddr)
		price := big.NewInt(1)

		tx, err = client.Transcoder(eth.FromPerc(10), eth.FromPerc(5), price)
		if err == eth.ErrCurrentRoundLocked {
			// wait for next round and retry
		}
		if err != nil {
			glog.Error(err)
			return
		}

		err = client.CheckTx(tx)
		if err != nil {
			glog.Error(err)
			return
		}

		glog.Infof("Storing service URI %v in service registry...", serviceURI)

		tx, err = client.SetServiceURI(serviceURI)
		if err != nil {
			glog.Error(err)
			return
		}

		err = client.CheckTx(tx)
		if err != nil {
			glog.Error(err)
		}
	}
}

func createRunScript(ethAcctAddr, dataDir string, isBroadcaster bool) {
	script := "#!/bin/bash\n"
	script += fmt.Sprintf(`./livepeer -v 99 -controllerAddr %s -datadir ./%s \
    -ethAcctAddr %s \
    -ethUrl %s \
    -ethPassword "" \
    -gasPrice 200 -gasLimit 2000000 \
    -monitor=false -currentManifest=true `,
		controllerAddr, dataDir, ethAcctAddr, endpoint)

	if !isBroadcaster {
		script += fmt.Sprintf(` -initializeRound=true \
    -serviceAddr 127.0.0.1:8936 -httpAddr 127.0.0.1:8936  -transcoder \
    -cliAddr 127.0.0.1:7936 -ipfsPath ./%s/trans
    `, dataDir)
	}

	glog.Info(script)
	fName := fmt.Sprintf("run_%s_%s.sh", getNodeType(isBroadcaster), ethAcctAddr)
	err := ioutil.WriteFile(fName, []byte(script), 0755)
	if err != nil {
		glog.Warningf("Error writing run script: %v", err)
	}
}

func createKey(keystoreDir string) string {
	keyStore := keystore.NewKeyStore(keystoreDir, keystore.StandardScryptN, keystore.StandardScryptP)

	account, err := keyStore.NewAccount(passphrase)
	if err != nil {
		glog.Fatal(err)
	}
	glog.Infof("Using ETH account: %v", account.Address.Hex())
	return account.Address.Hex()
}

func remoteConsole(destAccountAddr string) error {
	broadcasterGeth := "0161e041aad467a890839d5b08b138c1e6373072"
	if destAccountAddr != "" {
		broadcasterGeth = destAccountAddr
	}
	script := fmt.Sprintf("eth.sendTransaction({from: \"%s\", to: \"%s\", value: web3.toWei(834, \"ether\")})",
		gethMiningAccount, broadcasterGeth)

	client, err := rpc.Dial(endpoint)
	if err != nil {
		glog.Fatalf("Unable to attach to remote geth: %v", err)
	}
	tmp, err := ioutil.TempDir("", "console")
	if err != nil {
		glog.Fatalf("Can't create temporary directory: %v", err)
	}
	defer os.RemoveAll(tmp)
	config := console.Config{
		DataDir: tmp,
		Client:  client,
	}

	console, err := console.New(config)
	if err != nil {
		glog.Fatalf("Failed to start the JavaScript console: %v", err)
	}
	defer console.Stop(false)

	err = console.Evaluate(script)
	if err != nil {
		glog.Error(err)
	}

	time.Sleep(3 * time.Second)

	return err
}
<commit_msg>devtool - auto find controller address and mining account fixes #663<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"math/big"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/ethereum/go-ethereum/accounts/keystore"
	ethcommon "github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/console"
	"github.com/ethereum/go-ethereum/ethclient"
	"github.com/ethereum/go-ethereum/rpc"
	"github.com/livepeer/go-livepeer/eth"

	"github.com/golang/glog"
)

const (
	clientIdentifier = "geth" // Client identifier to advertise over the network
	passphrase       = ""
	serviceURI       = "https://127.0.0.1:8936"
)

var (
	ethTxTimeout              = 600 * time.Second
	endpoint                  = "ws://localhost:8546/"
	gethMiningAccount         = "87da6a8c6e9eff15d703fc2773e32f6af8dbe301"
	gethMiningAccountOverride = false
	controllerAddr            = "0x04B9De88c81cda06165CF65a908e5f1EFBB9493B"
	controllerAddrOverride    = false
)

func main() {
	flag.Set("logtostderr", "true")
	baseDataDir := flag.String("datadir", ".lpdev2", "default data directory")
	endpointAddr := flag.String("endpoint", "", "Geth endpoint to connect to")
	miningAccountFlag := flag.String("miningaccount", "", "Override geth mining account (usually not needed)")
	controllerAddrFlag := flag.String("controller", "", "Override controller address (usually not needed)")

	flag.Parse()
	if *endpointAddr != "" {
		endpoint = *endpointAddr
	}
	if *miningAccountFlag != "" {
		gethMiningAccount = *miningAccountFlag
		gethMiningAccountOverride = true
	}
	if *controllerAddrFlag != "" {
		controllerAddr = *controllerAddrFlag
		controllerAddrOverride = true
	}
	args := flag.Args()
	goodToGo := false
	isBroadcaster := true
	if len(args) > 1 && args[0] == "setup" {
		switch args[1] {
		case "broadcaster":
			goodToGo = true
		case "transcoder":
			isBroadcaster = false
			goodToGo = true
		}
	}
	if !goodToGo {
		fmt.Println(`
    Usage: go run cmd/devtool/devtool.go setup broadcaster|transcoder
        It will create initilize eth account (on private testnet) to be used for broadcaster or transcoder
        and will create shell script (run_broadcaster_ETHACC.sh or run_transcoder_ETHACC.sh) to run it.`)
		return
	}

	t := getNodeType(isBroadcaster)

	tmp, err := ioutil.TempDir("", "livepeer")
	if err != nil {
		glog.Fatalf("Can't create temporary directory: %v", err)
	}
	defer os.RemoveAll(tmp)

	tempKeystoreDir := filepath.Join(tmp, "keystore")
	acc := createKey(tempKeystoreDir)
	glog.Infof("Using account %s", acc)
	dataDir := filepath.Join(*baseDataDir, t+"_"+acc)
	dataDirToCreate := filepath.Join(dataDir, "devenv")
	err = os.MkdirAll(dataDirToCreate, 0755)
	if err != nil {
		glog.Fatalf("Can't create directory %v", err)
	}

	keystoreDir := filepath.Join(dataDirToCreate, "keystore")
	err = os.Rename(tempKeystoreDir, keystoreDir)
	if err != nil {
		glog.Fatal(err)
	}
	remoteConsole(acc)
	ethSetup(acc, keystoreDir, isBroadcaster)
	createRunScript(acc, dataDir, isBroadcaster)
	glog.Info("Finished")
}

func getNodeType(isBroadcaster bool) string {
	t := "broadcaster"
	if !isBroadcaster {
		t = "transcoder"
	}
	return t
}

func ethSetup(ethAcctAddr, keystoreDir string, isBroadcaster bool) {
	time.Sleep(3 * time.Second)
	//Set up eth client
	backend, err := ethclient.Dial(endpoint)
	if err != nil {
		glog.Errorf("Failed to connect to Ethereum client: %v", err)
		return
	}
	glog.Infof("Using controller address %s", controllerAddr)

	client, err := eth.NewClient(ethcommon.HexToAddress(ethAcctAddr), keystoreDir, backend,
		ethcommon.HexToAddress(controllerAddr), ethTxTimeout)
	if err != nil {
		glog.Errorf("Failed to create client: %v", err)
		return
	}

	var bigGasPrice *big.Int = big.NewInt(int64(200))
	// var bigGasPrice *big.Int = big.NewInt(int64(00000))

	err = client.Setup(passphrase, 1000000, bigGasPrice)
	if err != nil {
		glog.Fatalf("Failed to setup client: %v", err)
		return
	}
	glog.Infof("Requesting tokens from faucet")

	tx, err := client.Request()
	if err != nil {
		glog.Errorf("Error requesting tokens from faucet: %v", err)
		return
	}

	err = client.CheckTx(tx)
	if err != nil {
		glog.Errorf("Error requesting tokens from faucet: %v", err)
		return
	}
	glog.Info("Done requesting tokens.")
	time.Sleep(4 * time.Second)

	if !isBroadcaster {
		// XXX TODO curl -X "POST" http://localhost:$transcoderCliPort/initializeRound
		time.Sleep(3 * time.Second)
		for {
			currentRound, err := client.CurrentRound()
			if err != nil {
				glog.Errorf("Error getting current round: %v", err)
				return
			}
			if currentRound.Int64() > 1 {
				break
			}
			// first round is initialized and locked, need to wait
			glog.Info("Waiting will first round ended.")
			time.Sleep(4 * time.Second)
		}
		tx, err := client.InitializeRound()
		// ErrRoundInitialized
		if err != nil {
			if err.Error() != "ErrRoundInitialized" {
				glog.Errorf("Error initializing round: %v", err)
				return
			}
		} else {
			err = client.CheckTx(tx)
			if err != nil {
				glog.Errorf("Error initializng round: %v", err)
				return
			}
		}
		glog.Info("Done initializing round.")
		glog.Info("Activating transcoder")
		// curl -d "blockRewardCut=10&feeShare=5&pricePerSegment=1&amount=500" --data-urlencode "serviceURI=https://$transcoderServiceAddr" \
		//   -H "Content-Type: application/x-www-form-urlencoded" \
		//   -X "POST" http://localhost:$transcoderCliPort/activateTranscoder\
		var amount *big.Int = big.NewInt(int64(500))
		glog.Infof("Bonding %v to %s", amount, ethAcctAddr)

		tx, err = client.Bond(amount, ethcommon.HexToAddress(ethAcctAddr))
		if err != nil {
			glog.Error(err)
			return
		}

		err = client.CheckTx(tx)
		if err != nil {
			glog.Error(err)
			return
		}
		glog.Infof("Registering transcoder %v", ethAcctAddr)
		price := big.NewInt(1)

		tx, err = client.Transcoder(eth.FromPerc(10), eth.FromPerc(5), price)
		if err == eth.ErrCurrentRoundLocked {
			// wait for next round and retry
		}
		if err != nil {
			glog.Error(err)
			return
		}

		err = client.CheckTx(tx)
		if err != nil {
			glog.Error(err)
			return
		}

		glog.Infof("Storing service URI %v in service registry...", serviceURI)

		tx, err = client.SetServiceURI(serviceURI)
		if err != nil {
			glog.Error(err)
			return
		}

		err = client.CheckTx(tx)
		if err != nil {
			glog.Error(err)
		}
	}
}

func createRunScript(ethAcctAddr, dataDir string, isBroadcaster bool) {
	script := "#!/bin/bash\n"
	script += fmt.Sprintf(`./livepeer -v 99 -controllerAddr %s -datadir ./%s \
    -ethAcctAddr %s \
    -ethUrl %s \
    -ethPassword "" \
    -gasPrice 200 -gasLimit 2000000 -devenv=true \
    -monitor=false -currentManifest=true `,
		controllerAddr, dataDir, ethAcctAddr, endpoint)

	if !isBroadcaster {
		script += fmt.Sprintf(` -initializeRound=true \
    -serviceAddr 127.0.0.1:8936 -httpAddr 127.0.0.1:8936  -transcoder \
    -cliAddr 127.0.0.1:7936 -ipfsPath ./%s/trans
    `, dataDir)
	}

	glog.Info(script)
	fName := fmt.Sprintf("run_%s_%s.sh", getNodeType(isBroadcaster), ethAcctAddr)
	err := ioutil.WriteFile(fName, []byte(script), 0755)
	if err != nil {
		glog.Warningf("Error writing run script: %v", err)
	}
}

func createKey(keystoreDir string) string {
	keyStore := keystore.NewKeyStore(keystoreDir, keystore.StandardScryptN, keystore.StandardScryptP)

	account, err := keyStore.NewAccount(passphrase)
	if err != nil {
		glog.Fatal(err)
	}
	glog.Infof("Using ETH account: %v", account.Address.Hex())
	return account.Address.Hex()
}

func remoteConsole(destAccountAddr string) error {
	broadcasterGeth := "0161e041aad467a890839d5b08b138c1e6373072"
	if destAccountAddr != "" {
		broadcasterGeth = destAccountAddr
	}

	client, err := rpc.Dial(endpoint)
	if err != nil {
		glog.Fatalf("Unable to attach to remote geth: %v", err)
	}
	// get mining account
	if !gethMiningAccountOverride {
		var accounts []string
		err = client.Call(&accounts, "eth_accounts")
		if err != nil {
			glog.Fatalf("Error finding mining account: %v", err)
		}
		if len(accounts) == 0 {
			glog.Fatal("Can't find mining account")
		}
		gethMiningAccount = accounts[0]
		glog.Infof("Found mining account: %s", gethMiningAccount)
	}

	tmp, err := ioutil.TempDir("", "console")
	if err != nil {
		glog.Fatalf("Can't create temporary directory: %v", err)
	}
	defer os.RemoveAll(tmp)

	printer := new(bytes.Buffer)

	config := console.Config{
		DataDir: tmp,
		Client:  client,
		Printer: printer,
	}

	console, err := console.New(config)
	if err != nil {
		glog.Fatalf("Failed to start the JavaScript console: %v", err)
	}
	defer console.Stop(false)

	if !controllerAddrOverride {
		// f9a6cf519167d81bc5cb3d26c60c0c9a19704aa908c148e82a861b570f4cd2d7 - SetContractInfo event
		getControllerAddressScript := `
		var logs = [];
		var filter = web3.eth.filter({ fromBlock: 0, toBlock: "latest",
			topics: ["0xf9a6cf519167d81bc5cb3d26c60c0c9a19704aa908c148e82a861b570f4cd2d7"]});
		filter.get(function(error, result){
			logs.push(result);
		});
		console.log(logs[0][0].address);''
	`
		glog.Infof("Running eth script: %s", getControllerAddressScript)
		err = console.Evaluate(getControllerAddressScript)
		if err != nil {
			glog.Error(err)
		}
		if printer.Len() == 0 {
			glog.Fatal("Can't find deployed controller")
		}
		controllerAddr = strings.Split(printer.String(), "\n")[0]

		glog.Infof("Found controller address: %s", controllerAddr)
	}

	script := fmt.Sprintf("eth.sendTransaction({from: \"%s\", to: \"%s\", value: web3.toWei(834, \"ether\")})",
		gethMiningAccount, broadcasterGeth)
	glog.Infof("Running eth script: %s", script)

	err = console.Evaluate(script)
	if err != nil {
		glog.Error(err)
	}

	time.Sleep(3 * time.Second)

	return err
}
<|endoftext|>


<commit_before>package deployer

import (
	"context"
	"fmt"
	"reflect"
	"strings"

	"github.com/rancher/rancher/pkg/ref"
	"github.com/rancher/types/apis/core/v1"

	"github.com/rancher/norman/controller"
	alertutil "github.com/rancher/rancher/pkg/controllers/user/alert/common"
	"github.com/rancher/rancher/pkg/controllers/user/systemimage"
	monitorutil "github.com/rancher/rancher/pkg/monitoring"
	"github.com/rancher/types/apis/apps/v1beta2"
	"github.com/rancher/types/apis/management.cattle.io/v3"
	projectv3 "github.com/rancher/types/apis/project.cattle.io/v3"

	"github.com/rancher/rancher/pkg/controllers/user/helm/common"
	"github.com/rancher/rancher/pkg/namespace"
	"github.com/rancher/rancher/pkg/settings"
	"github.com/rancher/types/config"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
)

var (
	serviceName = "alerting"
)

type alertService struct {
	clusterName        string
	apps               projectv3.AppInterface
	deployments        v1beta2.DeploymentInterface
	oldClusterAlerts   v3.ClusterAlertInterface
	oldProjectAlerts   v3.ProjectAlertInterface
	clusterAlertGroups v3.ClusterAlertGroupInterface
	projectAlertGroups v3.ProjectAlertGroupInterface
	clusterAlertRules  v3.ClusterAlertRuleInterface
	projectAlertRules  v3.ProjectAlertRuleInterface
	projectLister      v3.ProjectLister
	namespaces         v1.NamespaceInterface
	appDeployer        *appDeployer
}

func init() {
	systemimage.RegisterSystemService(serviceName, &alertService{})
}

func (l *alertService) Init(ctx context.Context, cluster *config.UserContext) {
	ad := &appDeployer{
		appsGetter:       cluster.Management.Project,
		namespaces:       cluster.Management.Core.Namespaces(metav1.NamespaceAll),
		secrets:          cluster.Core.Secrets(metav1.NamespaceAll),
		templateVersions: cluster.Management.Management.CatalogTemplateVersions(namespace.GlobalNamespace),
	}

	l.clusterName = cluster.ClusterName
	l.deployments = cluster.Apps.Deployments("")
	l.oldClusterAlerts = cluster.Management.Management.ClusterAlerts(cluster.ClusterName)
	l.oldProjectAlerts = cluster.Management.Management.ProjectAlerts(metav1.NamespaceAll)
	l.clusterAlertGroups = cluster.Management.Management.ClusterAlertGroups(cluster.ClusterName)
	l.projectAlertGroups = cluster.Management.Management.ProjectAlertGroups(metav1.NamespaceAll)
	l.clusterAlertRules = cluster.Management.Management.ClusterAlertRules(cluster.ClusterName)
	l.projectAlertRules = cluster.Management.Management.ProjectAlertRules(metav1.NamespaceAll)
	l.projectLister = cluster.Management.Management.Projects(cluster.ClusterName).Controller().Lister()
	l.apps = cluster.Management.Project.Apps(metav1.NamespaceAll)
	l.appDeployer = ad

}

func (l *alertService) Version() (string, error) {
	catalogID := settings.SystemMonitoringCatalogID.Get()
	templateVersionID, _, err := common.ParseExternalID(catalogID)
	if err != nil {
		return "", fmt.Errorf("get system monitor catalog version failed, %v", err)
	}
	return templateVersionID, nil
}

func (l *alertService) Upgrade(currentVersion string) (string, error) {
	newCatalogID := settings.SystemMonitoringCatalogID.Get()

	NewVersion, _, err := common.ParseExternalID(newCatalogID)
	if currentVersion == NewVersion {
		return currentVersion, nil
	}

	appName, _ := monitorutil.ClusterAlertManagerInfo()
	//migrate legacy
	if !strings.Contains(currentVersion, "system-library-rancher-monitoring") {
		if err := l.migrateLegacyClusterAlert(); err != nil {
			return "", err
		}

		if err := l.migrateLegacyProjectAlert(); err != nil {
			return "", err
		}
	}

	//upgrade old app
	defaultSystemProjects, err := l.projectLister.List(metav1.NamespaceAll, labels.Set(systemProjectLabel).AsSelector())
	if err != nil {
		return "", fmt.Errorf("list system project failed, %v", err)
	}

	if len(defaultSystemProjects) == 0 {
		return "", fmt.Errorf("get system project failed")
	}

	systemProject := defaultSystemProjects[0]
	if systemProject == nil {
		return "", fmt.Errorf("get system project failed")
	}
	app, err := l.apps.GetNamespaced(systemProject.Name, appName, metav1.GetOptions{})
	if err != nil {
		if apierrors.IsNotFound(err) {
			return NewVersion, nil
		}
		return "", fmt.Errorf("get app %s:%s failed, %v", systemProject.Name, appName, err)
	}
	newApp := app.DeepCopy()
	newApp.Spec.ExternalID = newCatalogID

	if !reflect.DeepEqual(newApp, app) {
		if _, err = l.apps.Update(newApp); err != nil {
			return "", fmt.Errorf("update app %s:%s failed, %v", app.Namespace, app.Name, err)
		}
	}
	return NewVersion, nil
}

func (l *alertService) migrateLegacyClusterAlert() error {
	oldClusterAlert, err := l.oldClusterAlerts.List(metav1.ListOptions{})
	if err != nil {
		return fmt.Errorf("get old cluster alert failed, %s", err)
	}
	for _, v := range oldClusterAlert.Items {
		migrationGroupName := fmt.Sprintf("migrate-group-%s", v.Name)
		groupID := alertutil.GetGroupID(l.clusterName, migrationGroupName)

		name := fmt.Sprintf("migrate-%s", v.Name)
		newClusterRule := &v3.ClusterAlertRule{
			ObjectMeta: metav1.ObjectMeta{
				Name:      name,
				Namespace: l.clusterName,
			},
			Spec: v3.ClusterAlertRuleSpec{
				ClusterName: l.clusterName,
				GroupName:   groupID,
				CommonRuleField: v3.CommonRuleField{
					DisplayName: v.Spec.DisplayName,
					Severity:    v.Spec.Severity,
					TimingField: v3.TimingField{
						GroupIntervalSeconds:  v.Spec.InitialWaitSeconds,
						RepeatIntervalSeconds: v.Spec.RepeatIntervalSeconds,
					},
				},
			},
		}

		if v.Spec.TargetNode != nil {
			newClusterRule.Spec.NodeRule = &v3.NodeRule{
				NodeName:     v.Spec.TargetNode.NodeName,
				Selector:     v.Spec.TargetNode.Selector,
				Condition:    v.Spec.TargetNode.Condition,
				MemThreshold: v.Spec.TargetNode.MemThreshold,
				CPUThreshold: v.Spec.TargetNode.CPUThreshold,
			}
		}

		if v.Spec.TargetEvent != nil {
			newClusterRule.Spec.EventRule = &v3.EventRule{
				EventType:    v.Spec.TargetEvent.EventType,
				ResourceKind: v.Spec.TargetEvent.ResourceKind,
			}
		}

		if v.Spec.TargetSystemService != nil {
			newClusterRule.Spec.SystemServiceRule = &v3.SystemServiceRule{
				Condition: v.Spec.TargetSystemService.Condition,
			}
		}

		oldClusterRule, err := l.clusterAlertRules.Get(newClusterRule.Name, metav1.GetOptions{})
		if err != nil {
			if !apierrors.IsNotFound(err) {
				return fmt.Errorf("migrate %s:%s failed, get alert rule failed, %v", v.Namespace, v.Name, err)
			}

			if _, err = l.clusterAlertRules.Create(newClusterRule); err != nil && !apierrors.IsAlreadyExists(err) {
				return fmt.Errorf("migrate %s:%s failed, create alert rule failed, %v", v.Namespace, v.Name, err)
			}
		} else {
			updatedClusterRule := oldClusterRule.DeepCopy()
			updatedClusterRule.Spec = newClusterRule.Spec
			if _, err := l.clusterAlertRules.Update(updatedClusterRule); err != nil {
				return fmt.Errorf("migrate %s:%s failed, update alert rule failed, %v", v.Namespace, v.Name, err)
			}
		}
		legacyGroup := &v3.ClusterAlertGroup{
			ObjectMeta: metav1.ObjectMeta{
				Name:      migrationGroupName,
				Namespace: l.clusterName,
			},
			Spec: v3.ClusterGroupSpec{
				ClusterName: l.clusterName,
				CommonGroupField: v3.CommonGroupField{
					DisplayName: "Migrate group",
					Description: "Migrate alert from last version",
				},
				Recipients: v.Spec.Recipients,
			},
		}

		legacyGroup, err = l.clusterAlertGroups.Create(legacyGroup)
		if err != nil && !apierrors.IsAlreadyExists(err) {
			return fmt.Errorf("migrate failed, create alert group %s:%s failed, %v", l.clusterName, migrationGroupName, err)
		}
	}
	return nil
}

func (l *alertService) migrateLegacyProjectAlert() error {
	oldProjectAlert, err := l.oldProjectAlerts.List(metav1.ListOptions{})
	if err != nil {
		return fmt.Errorf("get old project alert failed, %s", err)
	}

	oldProjectAlertGroup := make(map[string][]*v3.ProjectAlert)
	for _, v := range oldProjectAlert.Items {
		if controller.ObjectInCluster(l.clusterName, v) {
			oldProjectAlertGroup[v.Spec.ProjectName] = append(oldProjectAlertGroup[v.Spec.ProjectName], &v)
		}
	}

	for projectID, oldAlerts := range oldProjectAlertGroup {
		_, projectName := ref.Parse(projectID)

		for _, v := range oldAlerts {
			migrationGroupName := fmt.Sprintf("migrate-group-%s", v.Name)
			groupID := alertutil.GetGroupID(projectName, migrationGroupName)

			migrationRuleName := fmt.Sprintf("migrate-rule-%s", v.Name)
			newProjectRule := &v3.ProjectAlertRule{
				ObjectMeta: metav1.ObjectMeta{
					Name:      migrationRuleName,
					Namespace: projectName,
				},
				Spec: v3.ProjectAlertRuleSpec{
					ProjectName: projectName,
					GroupName:   groupID,
					CommonRuleField: v3.CommonRuleField{
						DisplayName: v.Spec.DisplayName,
						Severity:    v.Spec.Severity,
						TimingField: v3.TimingField{
							GroupIntervalSeconds:  v.Spec.InitialWaitSeconds,
							RepeatIntervalSeconds: v.Spec.RepeatIntervalSeconds,
						},
					},
				},
			}

			if v.Spec.TargetPod != nil {
				newProjectRule.Spec.PodRule = &v3.PodRule{
					PodName:                v.Spec.TargetPod.PodName,
					Condition:              v.Spec.TargetPod.Condition,
					RestartTimes:           v.Spec.TargetPod.RestartTimes,
					RestartIntervalSeconds: v.Spec.TargetPod.RestartIntervalSeconds,
				}
			}

			if v.Spec.TargetWorkload != nil {
				newProjectRule.Spec.WorkloadRule = &v3.WorkloadRule{
					WorkloadID:          v.Spec.TargetWorkload.WorkloadID,
					Selector:            v.Spec.TargetWorkload.Selector,
					AvailablePercentage: v.Spec.TargetWorkload.AvailablePercentage,
				}
			}

			oldProjectRule, err := l.projectAlertRules.GetNamespaced(projectName, newProjectRule.Name, metav1.GetOptions{})
			if err != nil {
				if !apierrors.IsNotFound(err) {
					return fmt.Errorf("migrate %s:%s failed, get alert rule failed, %v", v.Namespace, v.Name, err)
				}

				if _, err = l.projectAlertRules.Create(newProjectRule); err != nil && !apierrors.IsAlreadyExists(err) {
					return fmt.Errorf("migrate %s:%s failed, create alert rule failed, %v", v.Namespace, v.Name, err)
				}
			} else {
				updatedProjectRule := oldProjectRule.DeepCopy()
				updatedProjectRule.Spec = newProjectRule.Spec
				if _, err := l.projectAlertRules.Update(updatedProjectRule); err != nil {
					return fmt.Errorf("migrate %s:%s failed, update alert rule failed, %v", v.Namespace, v.Name, err)
				}
			}

			legacyGroup := &v3.ProjectAlertGroup{
				ObjectMeta: metav1.ObjectMeta{
					Name:      migrationGroupName,
					Namespace: projectName,
				},
				Spec: v3.ProjectGroupSpec{
					ProjectName: projectID,
					CommonGroupField: v3.CommonGroupField{
						DisplayName: "Migrate group",
						Description: "Migrate alert from last version",
					},
					Recipients: v.Spec.Recipients,
				},
			}

			legacyGroup, err = l.projectAlertGroups.Create(legacyGroup)
			if err != nil && !apierrors.IsAlreadyExists(err) {
				return fmt.Errorf("create migrate alert group %s:%s failed, %v", legacyGroup.Namespace, legacyGroup.Name, err)
			}
		}
	}
	return nil
}
<commit_msg>Fix project alerting can't show in UI after migrated<commit_after>package deployer

import (
	"context"
	"fmt"
	"reflect"
	"strings"

	"github.com/rancher/rancher/pkg/ref"
	"github.com/rancher/types/apis/core/v1"

	"github.com/rancher/norman/controller"
	alertutil "github.com/rancher/rancher/pkg/controllers/user/alert/common"
	"github.com/rancher/rancher/pkg/controllers/user/systemimage"
	monitorutil "github.com/rancher/rancher/pkg/monitoring"
	"github.com/rancher/types/apis/apps/v1beta2"
	"github.com/rancher/types/apis/management.cattle.io/v3"
	projectv3 "github.com/rancher/types/apis/project.cattle.io/v3"

	"github.com/rancher/rancher/pkg/controllers/user/helm/common"
	"github.com/rancher/rancher/pkg/namespace"
	"github.com/rancher/rancher/pkg/settings"
	"github.com/rancher/types/config"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
)

var (
	serviceName = "alerting"
)

const (
	defaultGroupIntervalSeconds = 180
)

type alertService struct {
	clusterName        string
	apps               projectv3.AppInterface
	oldClusterAlerts   v3.ClusterAlertInterface
	oldProjectAlerts   v3.ProjectAlertInterface
	clusterAlertGroups v3.ClusterAlertGroupInterface
	projectAlertGroups v3.ProjectAlertGroupInterface
	clusterAlertRules  v3.ClusterAlertRuleInterface
	projectAlertRules  v3.ProjectAlertRuleInterface
	projectLister      v3.ProjectLister
	namespaces         v1.NamespaceInterface
	appDeployer        *appDeployer
}

func init() {
	systemimage.RegisterSystemService(serviceName, &alertService{})
}

func (l *alertService) Init(ctx context.Context, cluster *config.UserContext) {
	ad := &appDeployer{
		appsGetter:       cluster.Management.Project,
		namespaces:       cluster.Management.Core.Namespaces(metav1.NamespaceAll),
		secrets:          cluster.Core.Secrets(metav1.NamespaceAll),
		templateVersions: cluster.Management.Management.CatalogTemplateVersions(namespace.GlobalNamespace),
	}

	l.clusterName = cluster.ClusterName
	l.deployments = cluster.Apps.Deployments("")
	l.oldClusterAlerts = cluster.Management.Management.ClusterAlerts(cluster.ClusterName)
	l.oldProjectAlerts = cluster.Management.Management.ProjectAlerts(metav1.NamespaceAll)
	l.clusterAlertGroups = cluster.Management.Management.ClusterAlertGroups(cluster.ClusterName)
	l.projectAlertGroups = cluster.Management.Management.ProjectAlertGroups(metav1.NamespaceAll)
	l.clusterAlertRules = cluster.Management.Management.ClusterAlertRules(cluster.ClusterName)
	l.projectAlertRules = cluster.Management.Management.ProjectAlertRules(metav1.NamespaceAll)
	l.projectLister = cluster.Management.Management.Projects(cluster.ClusterName).Controller().Lister()
	l.apps = cluster.Management.Project.Apps(metav1.NamespaceAll)
	l.appDeployer = ad

}

func (l *alertService) Version() (string, error) {
	catalogID := settings.SystemMonitoringCatalogID.Get()
	templateVersionID, _, err := common.ParseExternalID(catalogID)
	if err != nil {
		return "", fmt.Errorf("get system monitor catalog version failed, %v", err)
	}
	return templateVersionID, nil
}

func (l *alertService) Upgrade(currentVersion string) (string, error) {
	newCatalogID := settings.SystemMonitoringCatalogID.Get()

	NewVersion, _, err := common.ParseExternalID(newCatalogID)
	if currentVersion == NewVersion {
		return currentVersion, nil
	}

	appName, _ := monitorutil.ClusterAlertManagerInfo()
	//migrate legacy
	if !strings.Contains(currentVersion, "system-library-rancher-monitoring") {
		if err := l.migrateLegacyClusterAlert(); err != nil {
			return "", err
		}

		if err := l.migrateLegacyProjectAlert(); err != nil {
			return "", err
		}
	}

	//upgrade old app
	defaultSystemProjects, err := l.projectLister.List(metav1.NamespaceAll, labels.Set(systemProjectLabel).AsSelector())
	if err != nil {
		return "", fmt.Errorf("list system project failed, %v", err)
	}

	if len(defaultSystemProjects) == 0 {
		return "", fmt.Errorf("get system project failed")
	}

	systemProject := defaultSystemProjects[0]
	if systemProject == nil {
		return "", fmt.Errorf("get system project failed")
	}
	app, err := l.apps.GetNamespaced(systemProject.Name, appName, metav1.GetOptions{})
	if err != nil {
		if apierrors.IsNotFound(err) {
			return NewVersion, nil
		}
		return "", fmt.Errorf("get app %s:%s failed, %v", systemProject.Name, appName, err)
	}
	newApp := app.DeepCopy()
	newApp.Spec.ExternalID = newCatalogID

	if !reflect.DeepEqual(newApp, app) {
		if _, err = l.apps.Update(newApp); err != nil {
			return "", fmt.Errorf("update app %s:%s failed, %v", app.Namespace, app.Name, err)
		}
	}
	return NewVersion, nil
}

func (l *alertService) migrateLegacyClusterAlert() error {
	oldClusterAlert, err := l.oldClusterAlerts.List(metav1.ListOptions{})
	if err != nil {
		return fmt.Errorf("get old cluster alert failed, %s", err)
	}
	for _, v := range oldClusterAlert.Items {
		migrationGroupName := fmt.Sprintf("migrate-group-%s", v.Name)
		groupID := alertutil.GetGroupID(l.clusterName, migrationGroupName)

		name := fmt.Sprintf("migrate-%s", v.Name)
		newClusterRule := &v3.ClusterAlertRule{
			ObjectMeta: metav1.ObjectMeta{
				Name:      name,
				Namespace: l.clusterName,
			},
			Spec: v3.ClusterAlertRuleSpec{
				ClusterName: l.clusterName,
				GroupName:   groupID,
				CommonRuleField: v3.CommonRuleField{
					DisplayName: v.Spec.DisplayName,
					Severity:    v.Spec.Severity,
					TimingField: v3.TimingField{
						GroupWaitSeconds:      v.Spec.InitialWaitSeconds,
						GroupIntervalSeconds:  defaultGroupIntervalSeconds,
						RepeatIntervalSeconds: v.Spec.RepeatIntervalSeconds,
					},
				},
			},
		}

		if v.Spec.TargetNode != nil {
			newClusterRule.Spec.NodeRule = &v3.NodeRule{
				NodeName:     v.Spec.TargetNode.NodeName,
				Selector:     v.Spec.TargetNode.Selector,
				Condition:    v.Spec.TargetNode.Condition,
				MemThreshold: v.Spec.TargetNode.MemThreshold,
				CPUThreshold: v.Spec.TargetNode.CPUThreshold,
			}
		}

		if v.Spec.TargetEvent != nil {
			newClusterRule.Spec.EventRule = &v3.EventRule{
				EventType:    v.Spec.TargetEvent.EventType,
				ResourceKind: v.Spec.TargetEvent.ResourceKind,
			}
		}

		if v.Spec.TargetSystemService != nil {
			newClusterRule.Spec.SystemServiceRule = &v3.SystemServiceRule{
				Condition: v.Spec.TargetSystemService.Condition,
			}
		}

		oldClusterRule, err := l.clusterAlertRules.Get(newClusterRule.Name, metav1.GetOptions{})
		if err != nil {
			if !apierrors.IsNotFound(err) {
				return fmt.Errorf("migrate %s:%s failed, get alert rule failed, %v", v.Namespace, v.Name, err)
			}

			if _, err = l.clusterAlertRules.Create(newClusterRule); err != nil && !apierrors.IsAlreadyExists(err) {
				return fmt.Errorf("migrate %s:%s failed, create alert rule failed, %v", v.Namespace, v.Name, err)
			}
		} else {
			updatedClusterRule := oldClusterRule.DeepCopy()
			updatedClusterRule.Spec = newClusterRule.Spec
			if _, err := l.clusterAlertRules.Update(updatedClusterRule); err != nil {
				return fmt.Errorf("migrate %s:%s failed, update alert rule failed, %v", v.Namespace, v.Name, err)
			}
		}
		legacyGroup := &v3.ClusterAlertGroup{
			ObjectMeta: metav1.ObjectMeta{
				Name:      migrationGroupName,
				Namespace: l.clusterName,
			},
			Spec: v3.ClusterGroupSpec{
				ClusterName: l.clusterName,
				CommonGroupField: v3.CommonGroupField{
					DisplayName: "Migrate group",
					Description: "Migrate alert from last version",
					TimingField: v3.TimingField{
						GroupWaitSeconds:      v.Spec.InitialWaitSeconds,
						GroupIntervalSeconds:  defaultGroupIntervalSeconds,
						RepeatIntervalSeconds: v.Spec.RepeatIntervalSeconds,
					},
				},
				Recipients: v.Spec.Recipients,
			},
		}

		legacyGroup, err = l.clusterAlertGroups.Create(legacyGroup)
		if err != nil && !apierrors.IsAlreadyExists(err) {
			return fmt.Errorf("migrate failed, create alert group %s:%s failed, %v", l.clusterName, migrationGroupName, err)
		}
	}
	return nil
}

func (l *alertService) migrateLegacyProjectAlert() error {
	oldProjectAlert, err := l.oldProjectAlerts.List(metav1.ListOptions{})
	if err != nil {
		return fmt.Errorf("get old project alert failed, %s", err)
	}

	oldProjectAlertGroup := make(map[string][]v3.ProjectAlert)
	for _, v := range oldProjectAlert.Items {
		if controller.ObjectInCluster(l.clusterName, v) {
			oldProjectAlertGroup[v.Spec.ProjectName] = append(oldProjectAlertGroup[v.Spec.ProjectName], v)
		}
	}

	for projectID, oldAlerts := range oldProjectAlertGroup {
		_, projectName := ref.Parse(projectID)

		for _, v := range oldAlerts {
			migrationGroupName := fmt.Sprintf("migrate-group-%s", v.Name)
			groupID := alertutil.GetGroupID(projectName, migrationGroupName)

			migrationRuleName := fmt.Sprintf("migrate-rule-%s", v.Name)
			newProjectRule := &v3.ProjectAlertRule{
				ObjectMeta: metav1.ObjectMeta{
					Name:      migrationRuleName,
					Namespace: projectName,
				},
				Spec: v3.ProjectAlertRuleSpec{
					ProjectName: projectID,
					GroupName:   groupID,
					CommonRuleField: v3.CommonRuleField{
						DisplayName: v.Spec.DisplayName,
						Severity:    v.Spec.Severity,
						TimingField: v3.TimingField{
							GroupWaitSeconds:      v.Spec.InitialWaitSeconds,
							GroupIntervalSeconds:  defaultGroupIntervalSeconds,
							RepeatIntervalSeconds: v.Spec.RepeatIntervalSeconds,
						},
					},
				},
			}

			if v.Spec.TargetPod != nil {
				newProjectRule.Spec.PodRule = &v3.PodRule{
					PodName:                v.Spec.TargetPod.PodName,
					Condition:              v.Spec.TargetPod.Condition,
					RestartTimes:           v.Spec.TargetPod.RestartTimes,
					RestartIntervalSeconds: v.Spec.TargetPod.RestartIntervalSeconds,
				}
			}

			if v.Spec.TargetWorkload != nil {
				newProjectRule.Spec.WorkloadRule = &v3.WorkloadRule{
					WorkloadID:          v.Spec.TargetWorkload.WorkloadID,
					Selector:            v.Spec.TargetWorkload.Selector,
					AvailablePercentage: v.Spec.TargetWorkload.AvailablePercentage,
				}
			}

			oldProjectRule, err := l.projectAlertRules.GetNamespaced(projectName, newProjectRule.Name, metav1.GetOptions{})
			if err != nil {
				if !apierrors.IsNotFound(err) {
					return fmt.Errorf("migrate %s:%s failed, get alert rule failed, %v", v.Namespace, v.Name, err)
				}

				if _, err = l.projectAlertRules.Create(newProjectRule); err != nil && !apierrors.IsAlreadyExists(err) {
					return fmt.Errorf("migrate %s:%s failed, create alert rule failed, %v", v.Namespace, v.Name, err)
				}
			} else {
				updatedProjectRule := oldProjectRule.DeepCopy()
				updatedProjectRule.Spec = newProjectRule.Spec
				if _, err := l.projectAlertRules.Update(updatedProjectRule); err != nil {
					return fmt.Errorf("migrate %s:%s failed, update alert rule failed, %v", v.Namespace, v.Name, err)
				}
			}

			legacyGroup := &v3.ProjectAlertGroup{
				ObjectMeta: metav1.ObjectMeta{
					Name:      migrationGroupName,
					Namespace: projectName,
				},
				Spec: v3.ProjectGroupSpec{
					ProjectName: projectID,
					CommonGroupField: v3.CommonGroupField{
						DisplayName: "Migrate group",
						Description: "Migrate alert from last version",
						TimingField: v3.TimingField{
							GroupWaitSeconds:      v.Spec.InitialWaitSeconds,
							GroupIntervalSeconds:  defaultGroupIntervalSeconds,
							RepeatIntervalSeconds: v.Spec.RepeatIntervalSeconds,
						},
					},
					Recipients: v.Spec.Recipients,
				},
			}

			legacyGroup, err = l.projectAlertGroups.Create(legacyGroup)
			if err != nil && !apierrors.IsAlreadyExists(err) {
				return fmt.Errorf("create migrate alert group %s:%s failed, %v", legacyGroup.Namespace, legacyGroup.Name, err)
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package commands

import (
	"context"
	"fmt"
	"io/ioutil"
	"os/exec"
	"sync"
	"time"

	"github.com/gofrs/uuid"
	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"

	"github.com/brocaar/lora-gateway-bridge/internal/config"
	"github.com/brocaar/lora-gateway-bridge/internal/integration"
	"github.com/brocaar/loraserver/api/gw"
	"github.com/brocaar/lorawan"
)

type command struct {
	Command              string
	MaxExecutionDuration time.Duration
}

var (
	mux sync.RWMutex

	commands map[string]command
)

// Setup configures the gateway commands.
func Setup(conf config.Config) error {
	mux.Lock()
	defer mux.Unlock()

	commands = make(map[string]command)

	for k, v := range conf.Commands.Commands {
		commands[k] = command{
			Command:              v.Command,
			MaxExecutionDuration: v.MaxExecutionDuration,
		}

		log.WithFields(log.Fields{
			"command":                k,
			"command_exec":           v.Command,
			"max_execution_duration": v.MaxExecutionDuration,
		}).Info("commands: configuring command")
	}

	go executeLoop()

	return nil
}

func executeLoop() {
	for cmd := range integration.GetIntegration().GetGatewayCommandExecRequestChan() {
		go func(cmd gw.GatewayCommandExecRequest) {
			executeCommand(cmd)
		}(cmd)
	}
}

func executeCommand(cmd gw.GatewayCommandExecRequest) {
	var gatewayID lorawan.EUI64
	copy(gatewayID[:], cmd.GatewayId)

	stdout, stderr, err := execute(cmd.Command, cmd.Stdin, cmd.Environment)
	resp := gw.GatewayCommandExecResponse{
		GatewayId: cmd.GatewayId,
		ExecId:    cmd.ExecId,
		Stdout:    stdout,
		Stderr:    stderr,
	}
	if err != nil {
		resp.Error = err.Error()
	}

	var id uuid.UUID

	if err := integration.GetIntegration().PublishEvent(gatewayID, "exec", id, &resp); err != nil {
		log.WithError(err).Error("commands: publish command execution event error")
	}
}

func execute(command string, stdin []byte, environment map[string]string) ([]byte, []byte, error) {
	mux.RLock()
	defer mux.RUnlock()

	cmd, ok := commands[command]
	if !ok {
		return nil, nil, errors.New("command does not exist")
	}

	cmdArgs, err := ParseCommandLine(cmd.Command)
	if err != nil {
		return nil, nil, errors.Wrap(err, "parse command error")
	}
	if len(cmdArgs) == 0 {
		return nil, nil, errors.New("no command is given")
	}

	log.WithFields(log.Fields{
		"command":                command,
		"exec":                   cmdArgs[0],
		"args":                   cmdArgs[1:],
		"max_execution_duration": cmd.MaxExecutionDuration,
	}).Info("commands: executing command")

	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(cmd.MaxExecutionDuration))
	defer cancel()

	cmdCtx := exec.CommandContext(ctx, cmdArgs[0], cmdArgs[1:]...)
	for k, v := range environment {
		cmdCtx.Env = append(cmdCtx.Env, fmt.Sprintf("%s=%s", k, v))
	}

	stdinPipe, err := cmdCtx.StdinPipe()
	if err != nil {
		return nil, nil, errors.Wrap(err, "get stdin pipe error")
	}

	stdoutPipe, err := cmdCtx.StdoutPipe()
	if err != nil {
		return nil, nil, errors.Wrap(err, "get stdout pipe error")
	}

	stderrPipe, err := cmdCtx.StderrPipe()
	if err != nil {
		return nil, nil, errors.Wrap(err, "get stderr pipe error")
	}

	go func() {
		defer stdinPipe.Close()
		if _, err := stdinPipe.Write(stdin); err != nil {
			log.WithError(err).Error("commands: write to stdin error")
		}
	}()

	if err := cmdCtx.Start(); err != nil {
		return nil, nil, errors.Wrap(err, "starting command error")
	}

	stdoutB, _ := ioutil.ReadAll(stdoutPipe)
	stderrB, _ := ioutil.ReadAll(stderrPipe)

	if err := cmdCtx.Wait(); err != nil {
		return nil, nil, errors.Wrap(err, "waiting for command to finish error")
	}

	return stdoutB, stderrB, nil
}

// source: https://stackoverflow.com/questions/34118732/parse-a-command-line-string-into-flags-and-arguments-in-golang
func ParseCommandLine(command string) ([]string, error) {
	var args []string
	state := "start"
	current := ""
	quote := "\""
	escapeNext := true
	for i := 0; i < len(command); i++ {
		c := command[i]

		if state == "quotes" {
			if string(c) != quote {
				current += string(c)
			} else {
				args = append(args, current)
				current = ""
				state = "start"
			}
			continue
		}

		if escapeNext {
			current += string(c)
			escapeNext = false
			continue
		}

		if c == '\\' {
			escapeNext = true
			continue
		}

		if c == '"' || c == '\'' {
			state = "quotes"
			quote = string(c)
			continue
		}

		if state == "arg" {
			if c == ' ' || c == '\t' {
				args = append(args, current)
				current = ""
				state = "start"
			} else {
				current += string(c)
			}
			continue
		}

		if c != ' ' && c != '\t' {
			state = "arg"
			current += string(c)
		}
	}

	if state == "quotes" {
		return []string{}, errors.New(fmt.Sprintf("Unclosed quote in command line: %s", command))
	}

	if current != "" {
		args = append(args, current)
	}

	return args, nil
}
<commit_msg>Make sure to kill potential sub-process on command exec.<commit_after>package commands

import (
	"fmt"
	"io/ioutil"
	"os/exec"
	"sync"
	"syscall"
	"time"

	"github.com/gofrs/uuid"
	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"

	"github.com/brocaar/lora-gateway-bridge/internal/config"
	"github.com/brocaar/lora-gateway-bridge/internal/integration"
	"github.com/brocaar/loraserver/api/gw"
	"github.com/brocaar/lorawan"
)

type command struct {
	Command              string
	MaxExecutionDuration time.Duration
}

var (
	mux sync.RWMutex

	commands map[string]command
)

// Setup configures the gateway commands.
func Setup(conf config.Config) error {
	mux.Lock()
	defer mux.Unlock()

	commands = make(map[string]command)

	for k, v := range conf.Commands.Commands {
		commands[k] = command{
			Command:              v.Command,
			MaxExecutionDuration: v.MaxExecutionDuration,
		}

		log.WithFields(log.Fields{
			"command":                k,
			"command_exec":           v.Command,
			"max_execution_duration": v.MaxExecutionDuration,
		}).Info("commands: configuring command")
	}

	go executeLoop()

	return nil
}

func executeLoop() {
	for cmd := range integration.GetIntegration().GetGatewayCommandExecRequestChan() {
		go func(cmd gw.GatewayCommandExecRequest) {
			executeCommand(cmd)
		}(cmd)
	}
}

func executeCommand(cmd gw.GatewayCommandExecRequest) {
	var gatewayID lorawan.EUI64
	copy(gatewayID[:], cmd.GatewayId)

	stdout, stderr, err := execute(cmd.Command, cmd.Stdin, cmd.Environment)
	resp := gw.GatewayCommandExecResponse{
		GatewayId: cmd.GatewayId,
		ExecId:    cmd.ExecId,
		Stdout:    stdout,
		Stderr:    stderr,
	}
	if err != nil {
		resp.Error = err.Error()
	}

	var id uuid.UUID

	if err := integration.GetIntegration().PublishEvent(gatewayID, "exec", id, &resp); err != nil {
		log.WithError(err).Error("commands: publish command execution event error")
	}
}

func execute(command string, stdin []byte, environment map[string]string) ([]byte, []byte, error) {
	mux.RLock()
	defer mux.RUnlock()

	cmdConfig, ok := commands[command]
	if !ok {
		return nil, nil, errors.New("command does not exist")
	}

	cmdArgs, err := ParseCommandLine(cmdConfig.Command)
	if err != nil {
		return nil, nil, errors.Wrap(err, "parse command error")
	}
	if len(cmdArgs) == 0 {
		return nil, nil, errors.New("no command is given")
	}

	log.WithFields(log.Fields{
		"command":                command,
		"exec":                   cmdArgs[0],
		"args":                   cmdArgs[1:],
		"max_execution_duration": cmdConfig.MaxExecutionDuration,
	}).Info("commands: executing command")

	cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...)
	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}

	for k, v := range environment {
		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v))
	}

	stdinPipe, err := cmd.StdinPipe()
	if err != nil {
		return nil, nil, errors.Wrap(err, "get stdin pipe error")
	}

	stdoutPipe, err := cmd.StdoutPipe()
	if err != nil {
		return nil, nil, errors.Wrap(err, "get stdout pipe error")
	}

	stderrPipe, err := cmd.StderrPipe()
	if err != nil {
		return nil, nil, errors.Wrap(err, "get stderr pipe error")
	}

	go func() {
		defer stdinPipe.Close()
		if _, err := stdinPipe.Write(stdin); err != nil {
			log.WithError(err).Error("commands: write to stdin error")
		}
	}()

	if err := cmd.Start(); err != nil {
		return nil, nil, errors.Wrap(err, "starting command error")
	}

	time.AfterFunc(cmdConfig.MaxExecutionDuration, func() {
		pgid, err := syscall.Getpgid(cmd.Process.Pid)
		if err == nil {
			if err := syscall.Kill(-pgid, syscall.SIGKILL); err != nil {
				panic(err)
			}
		}
	})

	stdoutB, _ := ioutil.ReadAll(stdoutPipe)
	stderrB, _ := ioutil.ReadAll(stderrPipe)

	if err := cmd.Wait(); err != nil {
		return nil, nil, errors.Wrap(err, "waiting for command to finish error")
	}

	return stdoutB, stderrB, nil
}

// source: https://stackoverflow.com/questions/34118732/parse-a-command-line-string-into-flags-and-arguments-in-golang
func ParseCommandLine(command string) ([]string, error) {
	var args []string
	state := "start"
	current := ""
	quote := "\""
	escapeNext := true
	for i := 0; i < len(command); i++ {
		c := command[i]

		if state == "quotes" {
			if string(c) != quote {
				current += string(c)
			} else {
				args = append(args, current)
				current = ""
				state = "start"
			}
			continue
		}

		if escapeNext {
			current += string(c)
			escapeNext = false
			continue
		}

		if c == '\\' {
			escapeNext = true
			continue
		}

		if c == '"' || c == '\'' {
			state = "quotes"
			quote = string(c)
			continue
		}

		if state == "arg" {
			if c == ' ' || c == '\t' {
				args = append(args, current)
				current = ""
				state = "start"
			} else {
				current += string(c)
			}
			continue
		}

		if c != ' ' && c != '\t' {
			state = "arg"
			current += string(c)
		}
	}

	if state == "quotes" {
		return []string{}, errors.New(fmt.Sprintf("Unclosed quote in command line: %s", command))
	}

	if current != "" {
		args = append(args, current)
	}

	return args, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package main

import (
	"bytes"
	"crypto/sha256"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"sort"

	"github.com/mvdan/fdroidcl"
)

var cmdUpdate = &Command{
	UsageLine: "update",
	Short:     "Update the index",
}

func init() {
	cmdUpdate.Run = runUpdate
}

func runUpdate(args []string) {
	for _, r := range config.Repos {
		if !r.Enabled {
			continue
		}
		if err := r.updateIndex(); err != nil {
			log.Fatalf("Could not update index: %v", err)
		}
	}
}

func (r *repo) updateIndex() error {
	url := fmt.Sprintf("%s/%s", r.URL, "index.jar")
	if err := downloadEtag(url, indexPath(r.ID), nil); err != nil {
		return err
	}
	return nil
}

func (r *repo) loadIndex() (*fdroidcl.Index, error) {
	p := indexPath(r.ID)
	f, err := os.Open(p)
	if err != nil {
		return nil, fmt.Errorf("could not open index: %v", err)
	}
	stat, err := f.Stat()
	if err != nil {
		return nil, fmt.Errorf("could not stat index: %v", err)
	}
	//pubkey, err := hex.DecodeString(repoPubkey)
	//if err != nil {
	//	return nil, fmt.Errorf("could not decode public key: %v", err)
	//}
	index, err := fdroidcl.LoadIndexJar(f, stat.Size(), nil)
	if err != nil {
		return nil, fmt.Errorf("could not load index: %v", err)
	}
	return index, nil
}

func respEtag(resp *http.Response) string {
	etags, e := resp.Header["Etag"]
	if !e || len(etags) == 0 {
		return ""
	}
	return etags[0]
}

func downloadEtag(url, path string, sum []byte) error {
	fmt.Printf("Downloading %s... ", url)
	defer fmt.Println()
	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)

	etagPath := path + "-etag"
	if _, err := os.Stat(path); err == nil {
		etag, _ := ioutil.ReadFile(etagPath)
		req.Header.Add("If-None-Match", string(etag))
	}

	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusNotModified {
		fmt.Printf("not modified")
		return nil
	}
	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}
	defer f.Close()
	if sum == nil {
		_, err := io.Copy(f, resp.Body)
		if err != nil {
			return err
		}
	} else {
		data, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		got := sha256.Sum256(data)
		if !bytes.Equal(sum, got[:]) {
			return errors.New("sha256 mismatch")
		}
		if _, err := f.Write(data); err != nil {
			return err
		}
	}
	if err := ioutil.WriteFile(etagPath, []byte(respEtag(resp)), 0644); err != nil {
		return err
	}
	fmt.Printf("done")
	return nil
}

func indexPath(name string) string {
	return filepath.Join(mustData(), name+".jar")
}

func mustLoadIndexes() []fdroidcl.App {
	m := make(map[string]*fdroidcl.App)
	for _, r := range config.Repos {
		if !r.Enabled {
			continue
		}
		index, err := r.loadIndex()
		if err != nil {
			log.Fatalf("Error while loading %s: %v", r.ID, err)
		}
		for i := range index.Apps {
			app := index.Apps[i]
			orig, e := m[app.ID]
			if !e {
				m[app.ID] = &app
				continue
			}
			apks := append(orig.Apks, app.Apks...)
			// We use a stable sort so that repository order
			// (priority) is preserved amongst apks with the same
			// vercode on apps
			sort.Stable(fdroidcl.ApkList(apks))
			m[app.ID].Apks = apks
		}
	}
	apps := make([]fdroidcl.App, 0, len(m))
	for _, a := range m {
		apps = append(apps, *a)
	}
	sort.Sort(fdroidcl.AppList(apps))
	return apps
}
<commit_msg>cmd/fdroidcl: treat HTTP error codes<commit_after>// Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package main

import (
	"bytes"
	"crypto/sha256"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"sort"

	"github.com/mvdan/fdroidcl"
)

var cmdUpdate = &Command{
	UsageLine: "update",
	Short:     "Update the index",
}

func init() {
	cmdUpdate.Run = runUpdate
}

func runUpdate(args []string) {
	for _, r := range config.Repos {
		if !r.Enabled {
			continue
		}
		if err := r.updateIndex(); err != nil {
			log.Fatalf("Could not update index: %v", err)
		}
	}
}

func (r *repo) updateIndex() error {
	url := fmt.Sprintf("%s/%s", r.URL, "index.jar")
	if err := downloadEtag(url, indexPath(r.ID), nil); err != nil {
		return err
	}
	return nil
}

func (r *repo) loadIndex() (*fdroidcl.Index, error) {
	p := indexPath(r.ID)
	f, err := os.Open(p)
	if err != nil {
		return nil, fmt.Errorf("could not open index: %v", err)
	}
	stat, err := f.Stat()
	if err != nil {
		return nil, fmt.Errorf("could not stat index: %v", err)
	}
	//pubkey, err := hex.DecodeString(repoPubkey)
	//if err != nil {
	//	return nil, fmt.Errorf("could not decode public key: %v", err)
	//}
	index, err := fdroidcl.LoadIndexJar(f, stat.Size(), nil)
	if err != nil {
		return nil, fmt.Errorf("could not load index: %v", err)
	}
	return index, nil
}

func respEtag(resp *http.Response) string {
	etags, e := resp.Header["Etag"]
	if !e || len(etags) == 0 {
		return ""
	}
	return etags[0]
}

func downloadEtag(url, path string, sum []byte) error {
	fmt.Printf("Downloading %s... ", url)
	defer fmt.Println()
	client := &http.Client{}
	req, err := http.NewRequest("GET", url, nil)

	etagPath := path + "-etag"
	if _, err := os.Stat(path); err == nil {
		etag, _ := ioutil.ReadFile(etagPath)
		req.Header.Add("If-None-Match", string(etag))
	}

	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode >= 400 {
		return fmt.Errorf("download failed: %d %s",
			resp.StatusCode, http.StatusText(resp.StatusCode))
	}
	if resp.StatusCode == http.StatusNotModified {
		fmt.Printf("not modified")
		return nil
	}
	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}
	defer f.Close()
	if sum == nil {
		_, err := io.Copy(f, resp.Body)
		if err != nil {
			return err
		}
	} else {
		data, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		got := sha256.Sum256(data)
		if !bytes.Equal(sum, got[:]) {
			return fmt.Errorf("sha256 mismatch")
		}
		if _, err := f.Write(data); err != nil {
			return err
		}
	}
	if err := ioutil.WriteFile(etagPath, []byte(respEtag(resp)), 0644); err != nil {
		return err
	}
	fmt.Printf("done")
	return nil
}

func indexPath(name string) string {
	return filepath.Join(mustData(), name+".jar")
}

func mustLoadIndexes() []fdroidcl.App {
	m := make(map[string]*fdroidcl.App)
	for _, r := range config.Repos {
		if !r.Enabled {
			continue
		}
		index, err := r.loadIndex()
		if err != nil {
			log.Fatalf("Error while loading %s: %v", r.ID, err)
		}
		for i := range index.Apps {
			app := index.Apps[i]
			orig, e := m[app.ID]
			if !e {
				m[app.ID] = &app
				continue
			}
			apks := append(orig.Apks, app.Apks...)
			// We use a stable sort so that repository order
			// (priority) is preserved amongst apks with the same
			// vercode on apps
			sort.Stable(fdroidcl.ApkList(apks))
			m[app.ID].Apks = apks
		}
	}
	apps := make([]fdroidcl.App, 0, len(m))
	for _, a := range m {
		apps = append(apps, *a)
	}
	sort.Sort(fdroidcl.AppList(apps))
	return apps
}
<|endoftext|>


<commit_before>package datapeer

import (
	"github.com/johnny-morrice/godless/api"
)

type residentMemoryStorage struct {
}

func MakeResidentMemoryStorage() api.ContentAddressableStorage {
	panic("not implemented")
}

type residentMemoryPubSubBus struct {
}

func MakeResidentMemoryPubSubBus() api.PubSubber {
	panic("not implemented")
}
<commit_msg>Implement in-memory pubsubber.<commit_after>package datapeer

import (
	"crypto"

	"github.com/johnny-morrice/godless/api"
)

type ResidentMemoryStorageOptions struct {
	Hash crypto.Hash
}

type residentMemoryStorage struct {
}

func MakeResidentMemoryStorage(options ResidentMemoryStorageOptions) api.ContentAddressableStorage {
	panic("not implemented")
}

type residentMemoryPubSubBus struct {
	bus []residentSubscription
}

func MakeResidentMemoryPubSubBus() api.PubSubber {
	return &residentMemoryPubSubBus{}
}

func (pubsubber *residentMemoryPubSubBus) PubSubPublish(topic, data string) error {
	for _, sub := range pubsubber.bus {
		subscription := sub
		if subscription.topic == topic {
			go subscription.publish(topic, data)
		}
	}

	return nil
}

func (pubsubber *residentMemoryPubSubBus) PubSubSubscribe(topic string) (api.PubSubSubscription, error) {
	subscription := residentSubscription{
		topic:  topic,
		nextch: make(chan api.PubSubRecord),
	}

	pubsubber.bus = append(pubsubber.bus, subscription)

	return subscription, nil
}

type residentSubscription struct {
	topic  string
	nextch chan api.PubSubRecord
}

func (subscription residentSubscription) publish(topic, data string) {
	record := residentPubSubRecord{
		data:   []byte(data),
		topics: []string{topic},
	}

	subscription.nextch <- record
}

func (subscription residentSubscription) Next() (api.PubSubRecord, error) {
	record := <-subscription.nextch
	return record, nil
}

type residentPubSubRecord struct {
	data   []byte
	topics []string
}

func (record residentPubSubRecord) From() string {
	return "Local Memory Peer"
}

func (record residentPubSubRecord) Data() []byte {
	return record.data
}

func (record residentPubSubRecord) SeqNo() int64 {
	return 0
}

// Not sure if TopicIds==Topics.
func (record residentPubSubRecord) TopicIDs() []string {
	return record.topics
}
<|endoftext|>


<commit_before>// Package errcheck is the library used to implement the errcheck command-line tool.
//
// Note: The API of this package has not been finalized and may change at any point.
package errcheck

import (
	"bufio"
	"errors"
	"fmt"
	"go/ast"
	"go/build"
	"go/token"
	"os"
	"regexp"
	"sort"
	"strings"
	"sync"

	"golang.org/x/tools/go/loader"
	"golang.org/x/tools/go/types"
)

var (
	// ErrNoGoFiles is returned when CheckPackage is run on a package with no Go source files
	ErrNoGoFiles = errors.New("package contains no go source files")
)

// UncheckedErrors is returned from the CheckPackage function if the package contains
// any unchecked errors.
type UncheckedErrors struct {
	// Errors is a list of all the unchecked errors in the package.
	// Printing an error reports its position within the file and the contents of the line.
	Errors []UncheckedError
}

func (e UncheckedErrors) Error() string {
	return fmt.Sprintf("%d unchecked errors", len(e.Errors))
}

// Len is the number of elements in the collection.
func (e UncheckedErrors) Len() int { return len(e.Errors) }

// Swap swaps the elements with indexes i and j.
func (e UncheckedErrors) Swap(i, j int) { e.Errors[i], e.Errors[j] = e.Errors[j], e.Errors[i] }

type byName struct{ UncheckedErrors }

// Less reports whether the element with index i should sort before the element with index j.
func (e byName) Less(i, j int) bool {
	ei, ej := e.Errors[i], e.Errors[j]

	pi, pj := ei.Pos, ej.Pos

	if pi.Filename != pj.Filename {
		return pi.Filename < pj.Filename
	}
	if pi.Line != pj.Line {
		return pi.Line < pj.Line
	}
	if pi.Column != pj.Column {
		return pi.Column < pj.Column
	}

	return ei.Line < ej.Line
}

type Checker struct {
	// ignore is a map of package names to regular expressions. Identifiers from a package are
	// checked against its regular expressions and if any of the expressions match the call
	// is not checked.
	Ignore map[string]*regexp.Regexp

	// If blank is true then assignments to the blank identifier are also considered to be
	// ignored errors.
	Blank bool

	// If asserts is true then ignored type assertion results are also checked
	Asserts bool

	// build tags
	Tags []string

	Verbose bool
}

func (c *Checker) logf(msg string, args ...interface{}) {
	if c.Verbose {
		fmt.Fprintf(os.Stderr, msg+"\n", args...)
	}
}

// CheckPackages checks packages for errors.
func (c *Checker) CheckPackages(paths ...string) error {
	ctx := build.Default
	for _, tag := range c.Tags {
		ctx.BuildTags = append(ctx.BuildTags, tag)
	}
	loadcfg := loader.Config{
		Build: &ctx,
	}
	rest, err := loadcfg.FromArgs(paths, true)
	if err != nil {
		return fmt.Errorf("could not parse arguments: %s", err)
	}
	if len(rest) > 0 {
		return fmt.Errorf("unhandled extra arguments: %v", rest)
	}

	program, err := loadcfg.Load()
	if err != nil {
		return fmt.Errorf("could not type check: %s", err)
	}

	var errsMutex sync.Mutex
	var errs []UncheckedError

	var wg sync.WaitGroup

	for _, pkgInfo := range program.InitialPackages() {
		if pkgInfo.Pkg.Path() == "unsafe" { // not a real package
			continue
		}

		wg.Add(1)

		go func(pkgInfo *loader.PackageInfo) {
			defer wg.Done()
			c.logf("Checking %s", pkgInfo.Pkg.Path())

			v := &visitor{
				prog:    program,
				pkg:     pkgInfo,
				ignore:  c.Ignore,
				blank:   c.Blank,
				asserts: c.Asserts,
				lines:   make(map[string][]string),
				errors:  []UncheckedError{},
			}

			for _, astFile := range v.pkg.Files {
				ast.Walk(v, astFile)
			}
			if len(v.errors) > 0 {
				errsMutex.Lock()
				defer errsMutex.Unlock()

				errs = append(errs, v.errors...)
			}
		}(pkgInfo)
	}

	wg.Wait()

	if len(errs) > 0 {
		u := UncheckedErrors{errs}

		sort.Sort(byName{u})

		return u
	}

	return nil
}

// visitor implements the errcheck algorithm
type visitor struct {
	prog    *loader.Program
	pkg     *loader.PackageInfo
	ignore  map[string]*regexp.Regexp
	blank   bool
	asserts bool
	lines   map[string][]string

	errors []UncheckedError
}

type UncheckedError struct {
	Pos  token.Position
	Line string
}

func (e UncheckedError) Error() string {
	return fmt.Sprintf("%s\t%s", e.Pos, e.Line)
}

func (v *visitor) ignoreCall(call *ast.CallExpr) bool {
	// Try to get an identifier.
	// Currently only supports simple expressions:
	//     1. f()
	//     2. x.y.f()
	var id *ast.Ident
	switch exp := call.Fun.(type) {
	case (*ast.Ident):
		id = exp
	case (*ast.SelectorExpr):
		id = exp.Sel
	default:
		// eg: *ast.SliceExpr, *ast.IndexExpr
	}

	if id == nil {
		return false
	}

	// If we got an identifier for the function, see if it is ignored
	if re, ok := v.ignore[""]; ok && re.MatchString(id.Name) {
		return true
	}

	if obj := v.pkg.Uses[id]; obj != nil {
		if pkg := obj.Pkg(); pkg != nil {
			if re, ok := v.ignore[pkg.Path()]; ok {
				return re.MatchString(id.Name)
			}
		}
	}

	return false
}

// errorsByArg returns a slice s such that
// len(s) == number of return types of call
// s[i] == true iff return type at position i from left is an error type
func (v *visitor) errorsByArg(call *ast.CallExpr) []bool {
	switch t := v.pkg.Types[call].Type.(type) {
	case *types.Named:
		// Single return
		return []bool{isErrorType(t.Obj())}
	case *types.Tuple:
		// Multiple returns
		s := make([]bool, t.Len())
		for i := 0; i < t.Len(); i++ {
			nt, ok := t.At(i).Type().(*types.Named)
			s[i] = ok && isErrorType(nt.Obj())
		}
		return s
	}
	return []bool{false}
}

func (v *visitor) callReturnsError(call *ast.CallExpr) bool {
	if v.isRecover(call) {
		return true
	}
	for _, isError := range v.errorsByArg(call) {
		if isError {
			return true
		}
	}
	return false
}

// isRecover returns true if the given CallExpr is a call to the built-in recover() function.
func (v *visitor) isRecover(call *ast.CallExpr) bool {
	if fun, ok := call.Fun.(*ast.Ident); ok {
		if _, ok := v.pkg.Uses[fun].(*types.Builtin); ok {
			return fun.Name == "recover"
		}
	}
	return false
}

func (v *visitor) addErrorAtPosition(position token.Pos) {
	pos := v.prog.Fset.Position(position)
	lines, ok := v.lines[pos.Filename]
	if !ok {
		lines = readfile(pos.Filename)
		v.lines[pos.Filename] = lines
	}

	line := "??"
	if pos.Line-1 < len(lines) {
		line = strings.TrimSpace(lines[pos.Line-1])
	}
	v.errors = append(v.errors, UncheckedError{pos, line})
}

func readfile(filename string) []string {
	var f, err = os.Open(filename)
	if err != nil {
		return nil
	}

	var lines []string
	var scanner = bufio.NewScanner(f)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}
	return lines
}

func (v *visitor) Visit(node ast.Node) ast.Visitor {
	switch stmt := node.(type) {
	case *ast.ExprStmt:
		if call, ok := stmt.X.(*ast.CallExpr); ok {
			if !v.ignoreCall(call) && v.callReturnsError(call) {
				v.addErrorAtPosition(call.Lparen)
			}
		}
	case *ast.GoStmt:
		if !v.ignoreCall(stmt.Call) && v.callReturnsError(stmt.Call) {
			v.addErrorAtPosition(stmt.Call.Lparen)
		}
	case *ast.DeferStmt:
		if !v.ignoreCall(stmt.Call) && v.callReturnsError(stmt.Call) {
			v.addErrorAtPosition(stmt.Call.Lparen)
		}
	case *ast.AssignStmt:
		if len(stmt.Rhs) == 1 {
			// single value on rhs; check against lhs identifiers
			if call, ok := stmt.Rhs[0].(*ast.CallExpr); ok {
				if !v.blank {
					break
				}
				if v.ignoreCall(call) {
					break
				}
				isError := v.errorsByArg(call)
				for i := 0; i < len(stmt.Lhs); i++ {
					if id, ok := stmt.Lhs[i].(*ast.Ident); ok {
						// We shortcut calls to recover() because errorsByArg can't
						// check its return types for errors since it returns interface{}.
						if id.Name == "_" && (v.isRecover(call) || isError[i]) {
							v.addErrorAtPosition(id.NamePos)
						}
					}
				}
			} else if assert, ok := stmt.Rhs[0].(*ast.TypeAssertExpr); ok {
				if !v.asserts {
					break
				}
				if assert.Type == nil {
					// type switch
					break
				}
				if len(stmt.Lhs) < 2 {
					// assertion result not read
					v.addErrorAtPosition(stmt.Rhs[0].Pos())
				} else if id, ok := stmt.Lhs[1].(*ast.Ident); ok && v.blank && id.Name == "_" {
					// assertion result ignored
					v.addErrorAtPosition(id.NamePos)
				}
			}
		} else {
			// multiple value on rhs; in this case a call can't return
			// multiple values. Assume len(stmt.Lhs) == len(stmt.Rhs)
			for i := 0; i < len(stmt.Lhs); i++ {
				if id, ok := stmt.Lhs[i].(*ast.Ident); ok {
					if call, ok := stmt.Rhs[i].(*ast.CallExpr); ok {
						if !v.blank {
							continue
						}
						if v.ignoreCall(call) {
							continue
						}
						if id.Name == "_" && v.callReturnsError(call) {
							v.addErrorAtPosition(id.NamePos)
						}
					} else if assert, ok := stmt.Rhs[i].(*ast.TypeAssertExpr); ok {
						if !v.asserts {
							continue
						}
						if assert.Type == nil {
							// Shouldn't happen anyway, no multi assignment in type switches
							continue
						}
						v.addErrorAtPosition(id.NamePos)
					}
				}
			}
		}
	default:
	}
	return v
}

type obj interface {
	Pkg() *types.Package
	Name() string
}

func isErrorType(v obj) bool {
	return v.Pkg() == nil && v.Name() == "error"
}
<commit_msg>Remove UncheckedError.Error<commit_after>// Package errcheck is the library used to implement the errcheck command-line tool.
//
// Note: The API of this package has not been finalized and may change at any point.
package errcheck

import (
	"bufio"
	"errors"
	"fmt"
	"go/ast"
	"go/build"
	"go/token"
	"os"
	"regexp"
	"sort"
	"strings"
	"sync"

	"golang.org/x/tools/go/loader"
	"golang.org/x/tools/go/types"
)

var (
	// ErrNoGoFiles is returned when CheckPackage is run on a package with no Go source files
	ErrNoGoFiles = errors.New("package contains no go source files")
)

// UncheckedErrors is returned from the CheckPackage function if the package contains
// any unchecked errors.
type UncheckedErrors struct {
	// Errors is a list of all the unchecked errors in the package.
	// Printing an error reports its position within the file and the contents of the line.
	Errors []UncheckedError
}

func (e UncheckedErrors) Error() string {
	return fmt.Sprintf("%d unchecked errors", len(e.Errors))
}

// Len is the number of elements in the collection.
func (e UncheckedErrors) Len() int { return len(e.Errors) }

// Swap swaps the elements with indexes i and j.
func (e UncheckedErrors) Swap(i, j int) { e.Errors[i], e.Errors[j] = e.Errors[j], e.Errors[i] }

type byName struct{ UncheckedErrors }

// Less reports whether the element with index i should sort before the element with index j.
func (e byName) Less(i, j int) bool {
	ei, ej := e.Errors[i], e.Errors[j]

	pi, pj := ei.Pos, ej.Pos

	if pi.Filename != pj.Filename {
		return pi.Filename < pj.Filename
	}
	if pi.Line != pj.Line {
		return pi.Line < pj.Line
	}
	if pi.Column != pj.Column {
		return pi.Column < pj.Column
	}

	return ei.Line < ej.Line
}

type Checker struct {
	// ignore is a map of package names to regular expressions. Identifiers from a package are
	// checked against its regular expressions and if any of the expressions match the call
	// is not checked.
	Ignore map[string]*regexp.Regexp

	// If blank is true then assignments to the blank identifier are also considered to be
	// ignored errors.
	Blank bool

	// If asserts is true then ignored type assertion results are also checked
	Asserts bool

	// build tags
	Tags []string

	Verbose bool
}

func (c *Checker) logf(msg string, args ...interface{}) {
	if c.Verbose {
		fmt.Fprintf(os.Stderr, msg+"\n", args...)
	}
}

// CheckPackages checks packages for errors.
func (c *Checker) CheckPackages(paths ...string) error {
	ctx := build.Default
	for _, tag := range c.Tags {
		ctx.BuildTags = append(ctx.BuildTags, tag)
	}
	loadcfg := loader.Config{
		Build: &ctx,
	}
	rest, err := loadcfg.FromArgs(paths, true)
	if err != nil {
		return fmt.Errorf("could not parse arguments: %s", err)
	}
	if len(rest) > 0 {
		return fmt.Errorf("unhandled extra arguments: %v", rest)
	}

	program, err := loadcfg.Load()
	if err != nil {
		return fmt.Errorf("could not type check: %s", err)
	}

	var errsMutex sync.Mutex
	var errs []UncheckedError

	var wg sync.WaitGroup

	for _, pkgInfo := range program.InitialPackages() {
		if pkgInfo.Pkg.Path() == "unsafe" { // not a real package
			continue
		}

		wg.Add(1)

		go func(pkgInfo *loader.PackageInfo) {
			defer wg.Done()
			c.logf("Checking %s", pkgInfo.Pkg.Path())

			v := &visitor{
				prog:    program,
				pkg:     pkgInfo,
				ignore:  c.Ignore,
				blank:   c.Blank,
				asserts: c.Asserts,
				lines:   make(map[string][]string),
				errors:  []UncheckedError{},
			}

			for _, astFile := range v.pkg.Files {
				ast.Walk(v, astFile)
			}
			if len(v.errors) > 0 {
				errsMutex.Lock()
				defer errsMutex.Unlock()

				errs = append(errs, v.errors...)
			}
		}(pkgInfo)
	}

	wg.Wait()

	if len(errs) > 0 {
		u := UncheckedErrors{errs}

		sort.Sort(byName{u})

		return u
	}

	return nil
}

// visitor implements the errcheck algorithm
type visitor struct {
	prog    *loader.Program
	pkg     *loader.PackageInfo
	ignore  map[string]*regexp.Regexp
	blank   bool
	asserts bool
	lines   map[string][]string

	errors []UncheckedError
}

type UncheckedError struct {
	Pos  token.Position
	Line string
}

func (v *visitor) ignoreCall(call *ast.CallExpr) bool {
	// Try to get an identifier.
	// Currently only supports simple expressions:
	//     1. f()
	//     2. x.y.f()
	var id *ast.Ident
	switch exp := call.Fun.(type) {
	case (*ast.Ident):
		id = exp
	case (*ast.SelectorExpr):
		id = exp.Sel
	default:
		// eg: *ast.SliceExpr, *ast.IndexExpr
	}

	if id == nil {
		return false
	}

	// If we got an identifier for the function, see if it is ignored
	if re, ok := v.ignore[""]; ok && re.MatchString(id.Name) {
		return true
	}

	if obj := v.pkg.Uses[id]; obj != nil {
		if pkg := obj.Pkg(); pkg != nil {
			if re, ok := v.ignore[pkg.Path()]; ok {
				return re.MatchString(id.Name)
			}
		}
	}

	return false
}

// errorsByArg returns a slice s such that
// len(s) == number of return types of call
// s[i] == true iff return type at position i from left is an error type
func (v *visitor) errorsByArg(call *ast.CallExpr) []bool {
	switch t := v.pkg.Types[call].Type.(type) {
	case *types.Named:
		// Single return
		return []bool{isErrorType(t.Obj())}
	case *types.Tuple:
		// Multiple returns
		s := make([]bool, t.Len())
		for i := 0; i < t.Len(); i++ {
			nt, ok := t.At(i).Type().(*types.Named)
			s[i] = ok && isErrorType(nt.Obj())
		}
		return s
	}
	return []bool{false}
}

func (v *visitor) callReturnsError(call *ast.CallExpr) bool {
	if v.isRecover(call) {
		return true
	}
	for _, isError := range v.errorsByArg(call) {
		if isError {
			return true
		}
	}
	return false
}

// isRecover returns true if the given CallExpr is a call to the built-in recover() function.
func (v *visitor) isRecover(call *ast.CallExpr) bool {
	if fun, ok := call.Fun.(*ast.Ident); ok {
		if _, ok := v.pkg.Uses[fun].(*types.Builtin); ok {
			return fun.Name == "recover"
		}
	}
	return false
}

func (v *visitor) addErrorAtPosition(position token.Pos) {
	pos := v.prog.Fset.Position(position)
	lines, ok := v.lines[pos.Filename]
	if !ok {
		lines = readfile(pos.Filename)
		v.lines[pos.Filename] = lines
	}

	line := "??"
	if pos.Line-1 < len(lines) {
		line = strings.TrimSpace(lines[pos.Line-1])
	}
	v.errors = append(v.errors, UncheckedError{pos, line})
}

func readfile(filename string) []string {
	var f, err = os.Open(filename)
	if err != nil {
		return nil
	}

	var lines []string
	var scanner = bufio.NewScanner(f)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}
	return lines
}

func (v *visitor) Visit(node ast.Node) ast.Visitor {
	switch stmt := node.(type) {
	case *ast.ExprStmt:
		if call, ok := stmt.X.(*ast.CallExpr); ok {
			if !v.ignoreCall(call) && v.callReturnsError(call) {
				v.addErrorAtPosition(call.Lparen)
			}
		}
	case *ast.GoStmt:
		if !v.ignoreCall(stmt.Call) && v.callReturnsError(stmt.Call) {
			v.addErrorAtPosition(stmt.Call.Lparen)
		}
	case *ast.DeferStmt:
		if !v.ignoreCall(stmt.Call) && v.callReturnsError(stmt.Call) {
			v.addErrorAtPosition(stmt.Call.Lparen)
		}
	case *ast.AssignStmt:
		if len(stmt.Rhs) == 1 {
			// single value on rhs; check against lhs identifiers
			if call, ok := stmt.Rhs[0].(*ast.CallExpr); ok {
				if !v.blank {
					break
				}
				if v.ignoreCall(call) {
					break
				}
				isError := v.errorsByArg(call)
				for i := 0; i < len(stmt.Lhs); i++ {
					if id, ok := stmt.Lhs[i].(*ast.Ident); ok {
						// We shortcut calls to recover() because errorsByArg can't
						// check its return types for errors since it returns interface{}.
						if id.Name == "_" && (v.isRecover(call) || isError[i]) {
							v.addErrorAtPosition(id.NamePos)
						}
					}
				}
			} else if assert, ok := stmt.Rhs[0].(*ast.TypeAssertExpr); ok {
				if !v.asserts {
					break
				}
				if assert.Type == nil {
					// type switch
					break
				}
				if len(stmt.Lhs) < 2 {
					// assertion result not read
					v.addErrorAtPosition(stmt.Rhs[0].Pos())
				} else if id, ok := stmt.Lhs[1].(*ast.Ident); ok && v.blank && id.Name == "_" {
					// assertion result ignored
					v.addErrorAtPosition(id.NamePos)
				}
			}
		} else {
			// multiple value on rhs; in this case a call can't return
			// multiple values. Assume len(stmt.Lhs) == len(stmt.Rhs)
			for i := 0; i < len(stmt.Lhs); i++ {
				if id, ok := stmt.Lhs[i].(*ast.Ident); ok {
					if call, ok := stmt.Rhs[i].(*ast.CallExpr); ok {
						if !v.blank {
							continue
						}
						if v.ignoreCall(call) {
							continue
						}
						if id.Name == "_" && v.callReturnsError(call) {
							v.addErrorAtPosition(id.NamePos)
						}
					} else if assert, ok := stmt.Rhs[i].(*ast.TypeAssertExpr); ok {
						if !v.asserts {
							continue
						}
						if assert.Type == nil {
							// Shouldn't happen anyway, no multi assignment in type switches
							continue
						}
						v.addErrorAtPosition(id.NamePos)
					}
				}
			}
		}
	default:
	}
	return v
}

type obj interface {
	Pkg() *types.Package
	Name() string
}

func isErrorType(v obj) bool {
	return v.Pkg() == nil && v.Name() == "error"
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"

	"github.com/akhenakh/gozim"
	"github.com/blevesearch/bleve"
	"github.com/blevesearch/bleve/registry"
)

type ArticleIndex struct {
	Title string
}

var (
	path       = flag.String("path", "", "path for the zim file")
	indexPath  = flag.String("index", "", "path for the index file")
	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
	z          *zim.ZimReader
	lang       = flag.String("lang", "en", "language for indexation")
)

func inList(s []string, value string) bool {
	for _, v := range s {
		if v == value {
			return true
		}
	}
	return false
}

// Type return the Article type (used for bleve indexer)
func (a *ArticleIndex) Type() string {
	return "Article"
}

func main() {

	flag.Parse()

	if *path == "" {
		panic("provide a zim file path")
	}

	z, err := zim.NewReader(*path, false)
	if err != nil {
		panic(err)
	}

	if *indexPath == "" {
		panic("Please provide a path for the index")
	}

	switch *lang {
	case "en":
		//TODO: create a simple language support for stop word
	default:
		panic("unsupported language")
	}

	mapping := bleve.NewIndexMapping()
	mapping.DefaultType = "Article"

	articleMapping := bleve.NewDocumentMapping()
	mapping.AddDocumentMapping("Article", articleMapping)

	titleMapping := bleve.NewTextFieldMapping()
	titleMapping.Store = false
	titleMapping.Index = true
	titleMapping.Analyzer = "standard"
	articleMapping.AddFieldMappingsAt("Title", titleMapping)

	fmt.Println(registry.AnalyzerTypesAndInstances())

	index, err := bleve.New(*indexPath, mapping)
	if err != nil {
		panic(err)
	}

	i := 0

	batch := bleve.NewBatch()
	batchCount := 0
	idoc := ArticleIndex{}

	z.ListTitlesPtrIterator(func(idx uint32) {

		if i == 10000 {
			fmt.Print("*")
			i = 0
		}

		offset := z.OffsetAtURLIdx(idx)
		a := z.ArticleAt(offset)
		if a.EntryType == zim.RedirectEntry || a.EntryType == zim.LinkTargetEntry || a.EntryType == zim.DeletedEntry {
			return
		}

		if a.Namespace == 'A' {
			idoc.Title = a.Title
			// index the idoc with the idx as key
			batch.Index(fmt.Sprint(idx), idoc)
		}

		batchCount++
		i++

		// send a batch to bleve
		if batchCount >= 10000 {
			err = index.Batch(batch)
			if err != nil {
				log.Fatal(err.Error())
			}
			batch = bleve.NewBatch()
			batchCount = 0
		}
	})

	// batch the rest
	if batchCount > 0 {
		err = index.Batch(batch)
		if err != nil {
			log.Fatal(err.Error())
		}
	}

	index.Close()

}
<commit_msg>closes #5 index redirect<commit_after>package main

import (
	"flag"
	"fmt"
	"log"

	"github.com/akhenakh/gozim"
	"github.com/blevesearch/bleve"
	"github.com/blevesearch/bleve/registry"
)

type ArticleIndex struct {
	Title string
}

var (
	path       = flag.String("path", "", "path for the zim file")
	indexPath  = flag.String("index", "", "path for the index file")
	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
	z          *zim.ZimReader
	lang       = flag.String("lang", "en", "language for indexation")
)

func inList(s []string, value string) bool {
	for _, v := range s {
		if v == value {
			return true
		}
	}
	return false
}

// Type return the Article type (used for bleve indexer)
func (a *ArticleIndex) Type() string {
	return "Article"
}

func main() {

	flag.Parse()

	if *path == "" {
		panic("provide a zim file path")
	}

	z, err := zim.NewReader(*path, false)
	if err != nil {
		panic(err)
	}

	if *indexPath == "" {
		panic("Please provide a path for the index")
	}

	switch *lang {
	case "en":
		//TODO: create a simple language support for stop word
	default:
		panic("unsupported language")
	}

	mapping := bleve.NewIndexMapping()
	mapping.DefaultType = "Article"

	articleMapping := bleve.NewDocumentMapping()
	mapping.AddDocumentMapping("Article", articleMapping)

	titleMapping := bleve.NewTextFieldMapping()
	titleMapping.Store = false
	titleMapping.Index = true
	titleMapping.Analyzer = "standard"
	articleMapping.AddFieldMappingsAt("Title", titleMapping)

	fmt.Println(registry.AnalyzerTypesAndInstances())

	index, err := bleve.New(*indexPath, mapping)
	if err != nil {
		panic(err)
	}

	i := 0

	batch := bleve.NewBatch()
	batchCount := 0
	idoc := ArticleIndex{}

	z.ListTitlesPtrIterator(func(idx uint32) {

		if i == 10000 {
			fmt.Print("*")
			i = 0
		}

		offset := z.OffsetAtURLIdx(idx)
		a := z.ArticleAt(offset)
		if a.EntryType == zim.DeletedEntry {
			i++
			return
		}

		if a.Namespace == 'A' {
			idoc.Title = a.Title
			// index the idoc with the idx as key
			batch.Index(fmt.Sprint(idx), idoc)
			batchCount++
		}

		i++

		// send a batch to bleve
		if batchCount >= 10000 {
			err = index.Batch(batch)
			if err != nil {
				log.Fatal(err.Error())
			}
			batch = bleve.NewBatch()
			batchCount = 0
		}
	})

	// batch the rest
	if batchCount > 0 {
		err = index.Batch(batch)
		if err != nil {
			log.Fatal(err.Error())
		}
	}

	index.Close()

}
<|endoftext|>