<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"

	"code.uber.internal/infra/kraken/core"
	"code.uber.internal/infra/kraken/lib/backend"
	"code.uber.internal/infra/kraken/lib/store"
	"code.uber.internal/infra/kraken/lib/torrent"
	"code.uber.internal/infra/kraken/lib/torrent/announcequeue"
	torrentstorage "code.uber.internal/infra/kraken/lib/torrent/storage"
	"code.uber.internal/infra/kraken/metrics"
	"code.uber.internal/infra/kraken/origin/blobclient"
	"code.uber.internal/infra/kraken/origin/blobserver"
	"code.uber.internal/infra/kraken/origin/torrentserver"
	"code.uber.internal/infra/kraken/tracker/announceclient"
	"code.uber.internal/infra/kraken/utils/configutil"
	"code.uber.internal/infra/kraken/utils/log"

	"github.com/andres-erbsen/clock"
)

func runTorrentServer(server *torrentserver.Server, port int) {
	addr := fmt.Sprintf(":%d", port)
	log.Infof("Starting torrent server on %s", addr)
	log.Fatal(http.ListenAndServe(addr, server.Handler()))
}

func runBlobServer(server *blobserver.Server, port int) {
	addr := fmt.Sprintf(":%d", port)
	log.Infof("Starting blob server on %d", addr)
	log.Fatal(http.ListenAndServe(addr, server.Handler()))
}

func main() {
	blobServerPort := flag.Int("blobserver_port", 0, "port which registry listens on")
	blobServerHostName := flag.String("blobserver_hostname", "", "optional hostname which blobserver will use to lookup a local host in a blob server hashnode config")
	peerIP := flag.String("peer_ip", "", "ip which peer will announce itself as")
	peerPort := flag.Int("peer_port", 0, "port which peer will announce itself as")
	configFile := flag.String("config", "", "Configuration file that has to be loaded from one of UBER_CONFIG_DIR locations")
	torrentServerPort := flag.Int("torrent_server_port", 0, "port which torrent server will listen on")
	zone := flag.String("zone", "", "zone/datacenter name")

	flag.Parse()

	if blobServerPort == nil || *blobServerPort == 0 {
		panic("0 is not a valid port for blob server")
	}
	if *torrentServerPort == 0 {
		panic("-torrent_server_port must be non-zero")
	}

	var config Config
	if err := configutil.Load(*configFile, &config); err != nil {
		panic(err)
	}

	zlog := log.ConfigureLogger(config.ZapLogging)
	defer zlog.Sync()

	stats, closer, err := metrics.New(config.Metrics)
	if err != nil {
		log.Fatalf("Failed to init metrics: %s", err)
	}
	defer closer.Close()

	fs, err := store.NewOriginFileStore(config.OriginStore, clock.New())
	if err != nil {
		log.Fatalf("Failed to create origin file store: %s", err)
	}

	pctx, err := core.NewPeerContext(
		core.PeerIDFactory(config.Torrent.PeerIDFactory), *zone, *peerIP, *peerPort, true)
	if err != nil {
		log.Fatalf("Failed to create peer context: %s", err)
	}

	c, err := torrent.NewSchedulerClient(
		config.Torrent,
		stats,
		pctx,
		announceclient.Disabled(),
		announcequeue.Disabled(),
		torrentstorage.NewOriginTorrentArchive(fs))
	if err != nil {
		log.Fatalf("Failed to create scheduler client: %s", err)
	}

	go runTorrentServer(torrentserver.New(c), *torrentServerPort)

	var hostname string
	if blobServerHostName == nil || *blobServerHostName == "" {
		hostname, err = os.Hostname()
		if err != nil {
			log.Fatalf("Error getting hostname: %s", err)
		}
	} else {
		hostname = *blobServerHostName
	}
	log.Infof("Configuring blob server with hostname '%s'", hostname)

	backendManager, err := backend.NewManager(config.Namespace)
	if err != nil {
		log.Fatalf("Error creating backend manager: %s", err)
	}

	bs, err := blobserver.New(
		config.BlobServer,
		stats,
		fmt.Sprintf("%s:%d", hostname, *blobServerPort),
		fs,
		blobclient.NewProvider(),
		pctx,
		backendManager)
	if err != nil {
		log.Fatalf("Error initializing blob server: %s", err)
	}
	go runBlobServer(bs, *blobServerPort)

	select {}
}
<commit_msg>Tag origin hostname on metrics<commit_after>package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"

	"code.uber.internal/infra/kraken/core"
	"code.uber.internal/infra/kraken/lib/backend"
	"code.uber.internal/infra/kraken/lib/store"
	"code.uber.internal/infra/kraken/lib/torrent"
	"code.uber.internal/infra/kraken/lib/torrent/announcequeue"
	torrentstorage "code.uber.internal/infra/kraken/lib/torrent/storage"
	"code.uber.internal/infra/kraken/metrics"
	"code.uber.internal/infra/kraken/origin/blobclient"
	"code.uber.internal/infra/kraken/origin/blobserver"
	"code.uber.internal/infra/kraken/origin/torrentserver"
	"code.uber.internal/infra/kraken/tracker/announceclient"
	"code.uber.internal/infra/kraken/utils/configutil"
	"code.uber.internal/infra/kraken/utils/log"

	"github.com/andres-erbsen/clock"
)

func runTorrentServer(server *torrentserver.Server, port int) {
	addr := fmt.Sprintf(":%d", port)
	log.Infof("Starting torrent server on %s", addr)
	log.Fatal(http.ListenAndServe(addr, server.Handler()))
}

func runBlobServer(server *blobserver.Server, port int) {
	addr := fmt.Sprintf(":%d", port)
	log.Infof("Starting blob server on %d", addr)
	log.Fatal(http.ListenAndServe(addr, server.Handler()))
}

func main() {
	blobServerPort := flag.Int("blobserver_port", 0, "port which registry listens on")
	blobServerHostName := flag.String("blobserver_hostname", "", "optional hostname which blobserver will use to lookup a local host in a blob server hashnode config")
	peerIP := flag.String("peer_ip", "", "ip which peer will announce itself as")
	peerPort := flag.Int("peer_port", 0, "port which peer will announce itself as")
	configFile := flag.String("config", "", "Configuration file that has to be loaded from one of UBER_CONFIG_DIR locations")
	torrentServerPort := flag.Int("torrent_server_port", 0, "port which torrent server will listen on")
	zone := flag.String("zone", "", "zone/datacenter name")

	flag.Parse()

	if blobServerPort == nil || *blobServerPort == 0 {
		panic("0 is not a valid port for blob server")
	}
	if *torrentServerPort == 0 {
		panic("-torrent_server_port must be non-zero")
	}

	var hostname string
	if blobServerHostName == nil || *blobServerHostName == "" {
		var err error
		hostname, err = os.Hostname()
		if err != nil {
			log.Fatalf("Error getting hostname: %s", err)
		}
	} else {
		hostname = *blobServerHostName
	}
	log.Infof("Configuring origin with hostname '%s'", hostname)

	var config Config
	if err := configutil.Load(*configFile, &config); err != nil {
		panic(err)
	}

	zlog := log.ConfigureLogger(config.ZapLogging)
	defer zlog.Sync()

	stats, closer, err := metrics.New(config.Metrics)
	if err != nil {
		log.Fatalf("Failed to init metrics: %s", err)
	}
	defer closer.Close()

	stats = stats.Tagged(map[string]string{
		"origin": hostname,
	})

	fs, err := store.NewOriginFileStore(config.OriginStore, clock.New())
	if err != nil {
		log.Fatalf("Failed to create origin file store: %s", err)
	}

	pctx, err := core.NewPeerContext(
		core.PeerIDFactory(config.Torrent.PeerIDFactory), *zone, *peerIP, *peerPort, true)
	if err != nil {
		log.Fatalf("Failed to create peer context: %s", err)
	}

	c, err := torrent.NewSchedulerClient(
		config.Torrent,
		stats,
		pctx,
		announceclient.Disabled(),
		announcequeue.Disabled(),
		torrentstorage.NewOriginTorrentArchive(fs))
	if err != nil {
		log.Fatalf("Failed to create scheduler client: %s", err)
	}

	go runTorrentServer(torrentserver.New(c), *torrentServerPort)

	backendManager, err := backend.NewManager(config.Namespace)
	if err != nil {
		log.Fatalf("Error creating backend manager: %s", err)
	}

	bs, err := blobserver.New(
		config.BlobServer,
		stats,
		fmt.Sprintf("%s:%d", hostname, *blobServerPort),
		fs,
		blobclient.NewProvider(),
		pctx,
		backendManager)
	if err != nil {
		log.Fatalf("Error initializing blob server: %s", err)
	}
	go runBlobServer(bs, *blobServerPort)

	select {}
}
<|endoftext|>


<commit_before>// These are types that conform to the Checker interface
// and can be assigned to a Check for watching service
// health.
package healthy

import (
	"errors"
	"net/http"
	"os/exec"
)

// A Checker that makes an HTTP get call and expects to get
// a 200-299 back as success. Anything else is considered
// a failure. The URL to hit is passed ass the args to the
// Run method.
type HttpGetCmd struct{}

func (h *HttpGetCmd) Run(args string) (int, error) {
	resp, err := http.Get(args)
	if resp == nil {
		return UNKNOWN, errors.New("No body from HTTP response!")
	}
	defer resp.Body.Close()

	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
		return HEALTHY, nil
	}

	return SICKLY, err
}

// A Checker that works with Nagios checks or other simple
// external tools. It expects a 0 exit code from the command
// that was run. Anything else is considered to be SICKLY.
// The command is passed as the args to the Run method. The
// command will be executed without a shell wrapper to keep
// the call as lean as possible in the majority case. If you
// need a shell you must invoke it yourself.
type ExternalCmd struct{}

func (e *ExternalCmd) Run(args string) (int, error) {
	cmd := exec.Command(args)
	err := cmd.Run()
	if err == nil {
		return HEALTHY, nil
	}

	return SICKLY, err
}
<commit_msg>Beef up the external command checker to do what we want.<commit_after>// These are types that conform to the Checker interface
// and can be assigned to a Check for watching service
// health.
package healthy

import (
	"errors"
	"log"
	"net/http"
	"os/exec"
	"strings"
)

// A Checker that makes an HTTP get call and expects to get
// a 200-299 back as success. Anything else is considered
// a failure. The URL to hit is passed ass the args to the
// Run method.
type HttpGetCmd struct{}

func (h *HttpGetCmd) Run(args string) (int, error) {
	resp, err := http.Get(args)
	if resp == nil {
		return UNKNOWN, errors.New("No body from HTTP response!")
	}
	defer resp.Body.Close()

	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
		return HEALTHY, nil
	}

	return SICKLY, err
}

// A Checker that works with Nagios checks or other simple
// external tools. It expects a 0 exit code from the command
// that was run. Anything else is considered to be SICKLY.
// The command is passed as the args to the Run method. The
// command will be executed without a shell wrapper to keep
// the call as lean as possible in the majority case. If you
// need a shell you must invoke it yourself.
type ExternalCmd struct{}

func (e *ExternalCmd) Run(args string) (int, error) {
	cliArgs := strings.Split(args, " ")
	cmd := exec.Command(cliArgs[0], cliArgs[1:]...)

	output, err := cmd.CombinedOutput()
	if err == nil {
		return HEALTHY, nil
	}

	log.Printf("Error running command: %s (%s)\n", err.Error(), output)
	return SICKLY, err
}
<|endoftext|>


<commit_before>// Copyright 2018 Anapaya Systems
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	_ "net/http/pprof"
	"os"
	"path/filepath"
	"time"

	"github.com/BurntSushi/toml"

	"github.com/scionproto/scion/go/lib/addr"
	"github.com/scionproto/scion/go/lib/common"
	"github.com/scionproto/scion/go/lib/env"
	"github.com/scionproto/scion/go/lib/fatal"
	"github.com/scionproto/scion/go/lib/infra"
	"github.com/scionproto/scion/go/lib/infra/infraenv"
	"github.com/scionproto/scion/go/lib/infra/modules/cleaner"
	"github.com/scionproto/scion/go/lib/infra/modules/itopo"
	"github.com/scionproto/scion/go/lib/infra/modules/trust"
	"github.com/scionproto/scion/go/lib/log"
	"github.com/scionproto/scion/go/lib/pathstorage"
	"github.com/scionproto/scion/go/lib/periodic"
	"github.com/scionproto/scion/go/lib/truststorage"
	"github.com/scionproto/scion/go/path_srv/internal/cryptosyncer"
	"github.com/scionproto/scion/go/path_srv/internal/handlers"
	"github.com/scionproto/scion/go/path_srv/internal/psconfig"
	"github.com/scionproto/scion/go/path_srv/internal/segsyncer"
	"github.com/scionproto/scion/go/proto"
)

type Config struct {
	General env.General
	Logging env.Logging
	Metrics env.Metrics
	TrustDB truststorage.TrustDBConf
	Infra   env.Infra
	PS      psconfig.Config
}

var (
	config      Config
	environment *env.Env
)

func init() {
	flag.Usage = env.Usage
}

// main initializes the path server and starts the dispatcher.
func main() {
	os.Exit(realMain())
}

func realMain() int {
	fatal.Init()
	env.AddFlags()
	flag.Parse()
	if v, ok := env.CheckFlags(psconfig.Sample); !ok {
		return v
	}
	if err := setupBasic(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return 1
	}
	defer log.Flush()
	defer env.LogAppStopped(common.PS, config.General.ID)
	defer log.LogPanicAndExit()
	if err := setup(); err != nil {
		log.Crit("Setup failed", "err", err)
		return 1
	}
	pathDB, revCache, err := pathstorage.NewPathStorage(config.PS.PathDB, config.PS.RevCache)
	if err != nil {
		log.Crit("Unable to initialize path storage", "err", err)
		return 1
	}
	trustDB, err := config.TrustDB.New()
	if err != nil {
		log.Crit("Unable to initialize trustDB", "err", err)
		return 1
	}
	topo := itopo.GetCurrentTopology()
	trustConf := &trust.Config{
		ServiceType: proto.ServiceType_ps,
	}
	trustStore, err := trust.NewStore(trustDB, topo.ISD_AS, trustConf, log.Root())
	if err != nil {
		log.Crit("Unable to initialize trust store", "err", err)
		return 1
	}
	err = trustStore.LoadAuthoritativeTRC(filepath.Join(config.General.ConfigDir, "certs"))
	if err != nil {
		log.Crit("TRC error", "err", err)
		return 1
	}
	topoAddress := topo.PS.GetById(config.General.ID)
	if topoAddress == nil {
		log.Crit("Unable to find topo address")
		return 1
	}
	msger, err := infraenv.InitMessenger(
		topo.ISD_AS,
		env.GetPublicSnetAddress(topo.ISD_AS, topoAddress),
		env.GetBindSnetAddress(topo.ISD_AS, topoAddress),
		addr.SvcPS,
		config.General.ReconnectToDispatcher,
		trustStore,
	)
	if err != nil {
		log.Crit(infraenv.ErrAppUnableToInitMessenger, "err", err)
		return 1
	}
	msger.AddHandler(infra.ChainRequest, trustStore.NewChainReqHandler(false))
	// TODO(lukedirtwalker): with the new CP-PKI design the PS should no longer need to handle TRC
	// and cert requests.
	msger.AddHandler(infra.TRCRequest, trustStore.NewTRCReqHandler(false))
	args := handlers.HandlerArgs{
		PathDB:     pathDB,
		RevCache:   revCache,
		TrustStore: trustStore,
		Config:     config.PS,
		IA:         topo.ISD_AS,
	}
	core := topo.Core
	var segReqHandler infra.Handler
	deduper := handlers.NewGetSegsDeduper(msger)
	if core {
		segReqHandler = handlers.NewSegReqCoreHandler(args, deduper)
	} else {
		segReqHandler = handlers.NewSegReqNonCoreHandler(args, deduper)
	}
	msger.AddHandler(infra.SegRequest, segReqHandler)
	msger.AddHandler(infra.SegReg, handlers.NewSegRegHandler(args))
	msger.AddHandler(infra.IfStateInfos, handlers.NewIfStatInfoHandler(args))
	if config.PS.SegSync && core {
		// Old down segment sync mechanism
		msger.AddHandler(infra.SegSync, handlers.NewSyncHandler(args))
		segSyncers, err := segsyncer.StartAll(args, msger)
		if err != nil {
			log.Crit("Unable to start seg syncer", "err", err)
			return 1
		}
		for _, segsync := range segSyncers {
			defer segsync.Stop()
		}
	}
	msger.AddHandler(infra.SegRev, handlers.NewRevocHandler(args))
	config.Metrics.StartPrometheus()
	// Start handling requests/messages
	go func() {
		defer log.LogPanicAndExit()
		msger.ListenAndServe()
	}()
	cleaner := periodic.StartPeriodicTask(cleaner.New(pathDB),
		periodic.NewTicker(300*time.Second), 295*time.Second)
	defer cleaner.Stop()
	cryptosyncer := periodic.StartPeriodicTask(&cryptosyncer.Syncer{
		DB:    trustDB,
		Msger: msger,
		IA:    topo.ISD_AS,
	}, periodic.NewTicker(30*time.Second), 30*time.Second)
	defer cryptosyncer.Stop()
	select {
	case <-environment.AppShutdownSignal:
		// Whenever we receive a SIGINT or SIGTERM we exit without an error.
		return 0
	case <-fatal.Chan():
		return 1
	}
}

func setupBasic() error {
	if _, err := toml.DecodeFile(env.ConfigFile(), &config); err != nil {
		return err
	}
	if err := env.InitLogging(&config.Logging); err != nil {
		return err
	}
	return env.LogAppStarted(common.PS, config.General.ID)
}

func setup() error {
	if err := env.InitGeneral(&config.General); err != nil {
		return err
	}
	itopo.SetCurrentTopology(config.General.Topology)
	environment = infraenv.InitInfraEnvironment(config.General.TopologyPath)
	config.PS.InitDefaults()
	return nil
}
<commit_msg>PS: Introduce struct for periodic tasks. (#2255)<commit_after>// Copyright 2018 Anapaya Systems
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	_ "net/http/pprof"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/BurntSushi/toml"

	"github.com/scionproto/scion/go/lib/addr"
	"github.com/scionproto/scion/go/lib/common"
	"github.com/scionproto/scion/go/lib/env"
	"github.com/scionproto/scion/go/lib/fatal"
	"github.com/scionproto/scion/go/lib/infra"
	"github.com/scionproto/scion/go/lib/infra/infraenv"
	"github.com/scionproto/scion/go/lib/infra/modules/cleaner"
	"github.com/scionproto/scion/go/lib/infra/modules/itopo"
	"github.com/scionproto/scion/go/lib/infra/modules/trust"
	"github.com/scionproto/scion/go/lib/infra/modules/trust/trustdb"
	"github.com/scionproto/scion/go/lib/log"
	"github.com/scionproto/scion/go/lib/pathstorage"
	"github.com/scionproto/scion/go/lib/periodic"
	"github.com/scionproto/scion/go/lib/truststorage"
	"github.com/scionproto/scion/go/path_srv/internal/cryptosyncer"
	"github.com/scionproto/scion/go/path_srv/internal/handlers"
	"github.com/scionproto/scion/go/path_srv/internal/psconfig"
	"github.com/scionproto/scion/go/path_srv/internal/segsyncer"
	"github.com/scionproto/scion/go/proto"
)

type Config struct {
	General env.General
	Logging env.Logging
	Metrics env.Metrics
	TrustDB truststorage.TrustDBConf
	Infra   env.Infra
	PS      psconfig.Config
}

var (
	config      Config
	environment *env.Env

	tasks *periodicTasks
)

func init() {
	flag.Usage = env.Usage
}

// main initializes the path server and starts the dispatcher.
func main() {
	os.Exit(realMain())
}

func realMain() int {
	fatal.Init()
	env.AddFlags()
	flag.Parse()
	if v, ok := env.CheckFlags(psconfig.Sample); !ok {
		return v
	}
	if err := setupBasic(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return 1
	}
	defer log.Flush()
	defer env.LogAppStopped(common.PS, config.General.ID)
	defer log.LogPanicAndExit()
	if err := setup(); err != nil {
		log.Crit("Setup failed", "err", err)
		return 1
	}
	pathDB, revCache, err := pathstorage.NewPathStorage(config.PS.PathDB, config.PS.RevCache)
	if err != nil {
		log.Crit("Unable to initialize path storage", "err", err)
		return 1
	}
	trustDB, err := config.TrustDB.New()
	if err != nil {
		log.Crit("Unable to initialize trustDB", "err", err)
		return 1
	}
	topo := itopo.GetCurrentTopology()
	trustConf := &trust.Config{
		ServiceType: proto.ServiceType_ps,
	}
	trustStore, err := trust.NewStore(trustDB, topo.ISD_AS, trustConf, log.Root())
	if err != nil {
		log.Crit("Unable to initialize trust store", "err", err)
		return 1
	}
	err = trustStore.LoadAuthoritativeTRC(filepath.Join(config.General.ConfigDir, "certs"))
	if err != nil {
		log.Crit("TRC error", "err", err)
		return 1
	}
	topoAddress := topo.PS.GetById(config.General.ID)
	if topoAddress == nil {
		log.Crit("Unable to find topo address")
		return 1
	}
	msger, err := infraenv.InitMessenger(
		topo.ISD_AS,
		env.GetPublicSnetAddress(topo.ISD_AS, topoAddress),
		env.GetBindSnetAddress(topo.ISD_AS, topoAddress),
		addr.SvcPS,
		config.General.ReconnectToDispatcher,
		trustStore,
	)
	if err != nil {
		log.Crit(infraenv.ErrAppUnableToInitMessenger, "err", err)
		return 1
	}
	msger.AddHandler(infra.ChainRequest, trustStore.NewChainReqHandler(false))
	// TODO(lukedirtwalker): with the new CP-PKI design the PS should no longer need to handle TRC
	// and cert requests.
	msger.AddHandler(infra.TRCRequest, trustStore.NewTRCReqHandler(false))
	args := handlers.HandlerArgs{
		PathDB:     pathDB,
		RevCache:   revCache,
		TrustStore: trustStore,
		Config:     config.PS,
		IA:         topo.ISD_AS,
	}
	core := topo.Core
	var segReqHandler infra.Handler
	deduper := handlers.NewGetSegsDeduper(msger)
	if core {
		segReqHandler = handlers.NewSegReqCoreHandler(args, deduper)
	} else {
		segReqHandler = handlers.NewSegReqNonCoreHandler(args, deduper)
	}
	msger.AddHandler(infra.SegRequest, segReqHandler)
	msger.AddHandler(infra.SegReg, handlers.NewSegRegHandler(args))
	msger.AddHandler(infra.IfStateInfos, handlers.NewIfStatInfoHandler(args))
	if config.PS.SegSync && core {
		// Old down segment sync mechanism
		msger.AddHandler(infra.SegSync, handlers.NewSyncHandler(args))
	}
	msger.AddHandler(infra.SegRev, handlers.NewRevocHandler(args))
	config.Metrics.StartPrometheus()
	// Start handling requests/messages
	go func() {
		defer log.LogPanicAndExit()
		msger.ListenAndServe()
	}()
	tasks = &periodicTasks{
		args:    args,
		msger:   msger,
		trustDB: trustDB,
	}
	tasks.Start()
	defer tasks.Kill()
	select {
	case <-environment.AppShutdownSignal:
		// Whenever we receive a SIGINT or SIGTERM we exit without an error.
		return 0
	case <-fatal.Chan():
		return 1
	}
}

type periodicTasks struct {
	args          handlers.HandlerArgs
	msger         infra.Messenger
	trustDB       trustdb.TrustDB
	mtx           sync.Mutex
	running       bool
	segSyncers    []*periodic.Runner
	pathDBCleaner *periodic.Runner
	cryptosyncer  *periodic.Runner
}

func (t *periodicTasks) Start() {
	fatal.Check()
	t.mtx.Lock()
	defer t.mtx.Unlock()
	if t.running {
		log.Warn("Trying to start task, but they are running! Ignored.")
		return
	}
	var err error
	if config.PS.SegSync && config.General.Topology.Core {
		t.segSyncers, err = segsyncer.StartAll(t.args, t.msger)
		if err != nil {
			fatal.Fatal(common.NewBasicError("Unable to start seg syncer", err))
		}
	}
	t.pathDBCleaner = periodic.StartPeriodicTask(cleaner.New(t.args.PathDB),
		periodic.NewTicker(300*time.Second), 295*time.Second)
	t.cryptosyncer = periodic.StartPeriodicTask(&cryptosyncer.Syncer{
		DB:    t.trustDB,
		Msger: t.msger,
		IA:    t.args.IA,
	}, periodic.NewTicker(30*time.Second), 30*time.Second)
	t.running = true
}

func (t *periodicTasks) Kill() {
	t.mtx.Lock()
	defer t.mtx.Unlock()
	if !t.running {
		log.Warn("Trying to stop tasks, but they are not running! Ignored.")
		return
	}
	for i := range t.segSyncers {
		syncer := t.segSyncers[i]
		syncer.Kill()
	}
	t.pathDBCleaner.Kill()
	t.cryptosyncer.Kill()
	t.running = false
}

func setupBasic() error {
	if _, err := toml.DecodeFile(env.ConfigFile(), &config); err != nil {
		return err
	}
	if err := env.InitLogging(&config.Logging); err != nil {
		return err
	}
	return env.LogAppStarted(common.PS, config.General.ID)
}

func setup() error {
	if err := env.InitGeneral(&config.General); err != nil {
		return err
	}
	itopo.SetCurrentTopology(config.General.Topology)
	environment = infraenv.InitInfraEnvironment(config.General.TopologyPath)
	config.PS.InitDefaults()
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2020 Anapaya Systems
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package ping implements pinging based on SCMP echo messages.
package ping

import (
	"context"
	"encoding/binary"
	"math/rand"
	"net"
	"time"

	"github.com/scionproto/scion/go/lib/addr"
	"github.com/scionproto/scion/go/lib/common"
	"github.com/scionproto/scion/go/lib/log"
	"github.com/scionproto/scion/go/lib/serrors"
	"github.com/scionproto/scion/go/lib/snet"
	"github.com/scionproto/scion/go/lib/sock/reliable"
	"github.com/scionproto/scion/go/lib/topology/underlay"
)

// Stats contains the statistics of a ping run.
type Stats struct {
	Sent     int
	Received int
}

// Update contains intermediary information about a received echo reply
type Update struct {
	Size     int
	Source   snet.SCIONAddress
	Sequence int
	RTT      time.Duration
	State    State
}

// State indicates the state of the echo reply
type State int

// Possible states.
const (
	Success State = iota
	AfterTimeout
	OutOfOrder
	Duplicate
)

// Config configures the ping run.
type Config struct {
	Dispatcher reliable.Dispatcher
	Local      *snet.UDPAddr
	Remote     *snet.UDPAddr

	// Attempts is the number of pings to send.
	Attempts uint16
	// Interval is the time between sending pings.
	Interval time.Duration
	// Timeout is the time until a ping is considered to have timed out.
	Timeout time.Duration
	// PayloadSize is the size of the SCMP echo payload.
	PayloadSize int

	// ErrHandler is invoked for every error that does not cause pinging to
	// abort. Execution time must be small, as it is run synchronously.
	ErrHandler func(err error)
	// Update handler is invoked for every ping reply. Execution time must be
	// small, as it is run synchronously.
	UpdateHandler func(Update)
}

// Run ping with the configuration. This blocks until the configured number
// attempts is sent, or the context is canceled.
func Run(ctx context.Context, cfg Config) (Stats, error) {
	if cfg.Interval < time.Millisecond {
		return Stats{}, serrors.New("interval below millisecond")
	}

	id := rand.Uint64()
	replies := make(chan reply, 10)

	svc := snet.DefaultPacketDispatcherService{
		Dispatcher: cfg.Dispatcher,
		SCMPHandler: scmpHandler{
			id:      uint16(id),
			replies: replies,
		},
	}
	conn, port, err := svc.Register(ctx, cfg.Local.IA, cfg.Local.Host, addr.SvcNone)
	if err != nil {
		return Stats{}, err
	}

	local := cfg.Local.Copy()
	local.Host.Port = int(port)

	// we need to have at least 8 bytes to store the request time in the
	// payload.
	if cfg.PayloadSize < 8 {
		cfg.PayloadSize = 8
	}
	p := pinger{
		attempts:      cfg.Attempts,
		interval:      cfg.Interval,
		timeout:       cfg.Timeout,
		pldSize:       cfg.PayloadSize,
		pld:           make([]byte, cfg.PayloadSize),
		id:            id,
		conn:          conn,
		local:         local,
		replies:       replies,
		errHandler:    cfg.ErrHandler,
		updateHandler: cfg.UpdateHandler,
	}
	return p.Ping(ctx, cfg.Remote)
}

type pinger struct {
	attempts uint16
	interval time.Duration
	timeout  time.Duration
	pldSize  int

	id      uint64
	conn    snet.PacketConn
	local   *snet.UDPAddr
	replies <-chan reply

	// Handlers
	errHandler    func(error)
	updateHandler func(Update)

	// Mutable state
	pld              []byte
	sentSequence     int
	receivedSequence int
	stats            Stats
}

func (p *pinger) Ping(ctx context.Context, remote *snet.UDPAddr) (Stats, error) {
	p.sentSequence, p.receivedSequence = -1, -1
	send := time.NewTicker(p.interval)
	defer send.Stop()

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	errSend := make(chan error, 1)

	go func() {
		defer log.HandlePanic()
		p.drain(ctx)
	}()

	go func() {
		defer log.HandlePanic()
		for i := uint16(0); i < p.attempts; i++ {
			if err := p.send(remote); err != nil {
				errSend <- serrors.WrapStr("sending", err)
				return
			}
			select {
			case <-send.C:
			case <-ctx.Done():
				return
			}
		}
		time.AfterFunc(p.timeout, cancel)
	}()

	for i := uint16(0); i < p.attempts; i++ {
		select {
		case <-ctx.Done():
			return p.stats, nil
		case err := <-errSend:
			return p.stats, err
		case reply := <-p.replies:
			if reply.Error != nil {
				if p.errHandler != nil {
					p.errHandler(reply.Error)
				}
				continue
			}
			p.receive(reply)
		}
	}
	return p.stats, nil
}

func (p *pinger) send(remote *snet.UDPAddr) error {
	sequence := p.sentSequence + 1

	binary.BigEndian.PutUint64(p.pld, uint64(time.Now().UnixNano()))
	pkt, err := pack(p.local, remote, snet.SCMPEchoRequest{
		Identifier: uint16(p.id),
		SeqNumber:  uint16(sequence),
		Payload:    p.pld,
	})
	if err != nil {
		return err
	}
	nextHop := remote.NextHop
	if nextHop == nil && p.local.IA.Equal(remote.IA) {
		nextHop = &net.UDPAddr{
			IP:   remote.Host.IP,
			Port: underlay.EndhostPort,
			Zone: remote.Host.Zone,
		}

	}
	if err := p.conn.WriteTo(pkt, nextHop); err != nil {
		return err
	}

	p.sentSequence = sequence
	p.stats.Sent++
	return nil
}

func (p *pinger) receive(reply reply) {
	rtt := reply.Received.Sub(time.Unix(0, int64(binary.BigEndian.Uint64(reply.Reply.Payload)))).
		Round(time.Microsecond)
	var state State
	switch {
	case rtt > p.timeout:
		state = AfterTimeout
	case int(reply.Reply.SeqNumber) < p.receivedSequence:
		state = OutOfOrder
	case int(reply.Reply.SeqNumber) == p.receivedSequence:
		state = Duplicate
	default:
		state = Success
		p.receivedSequence = int(reply.Reply.SeqNumber)
	}
	p.stats.Received++
	if p.updateHandler != nil {
		p.updateHandler(Update{
			RTT:      rtt,
			Sequence: int(reply.Reply.SeqNumber),
			Size:     reply.Size,
			Source:   reply.Source,
			State:    state,
		})
	}
}

func (p *pinger) drain(ctx context.Context) {
	var last time.Time
	for {
		select {
		case <-ctx.Done():
			return
		default:
			var pkt snet.Packet
			var ov net.UDPAddr
			if err := p.conn.ReadFrom(&pkt, &ov); err != nil && p.errHandler != nil {
				// Rate limit the error reports.
				if now := time.Now(); now.Sub(last) > 500*time.Millisecond {
					p.errHandler(serrors.WrapStr("reading packet", err))
					last = now
				}
			}
		}
	}
}

type reply struct {
	Received time.Time
	Source   snet.SCIONAddress
	Size     int
	Reply    snet.SCMPEchoReply
	Error    error
}

type scmpHandler struct {
	id      uint16
	replies chan<- reply
}

func (h scmpHandler) Handle(pkt *snet.Packet) error {
	echo, err := h.handle(pkt)
	h.replies <- reply{
		Received: time.Now(),
		Source:   pkt.Source,
		Size:     len(pkt.Bytes),
		Reply:    echo,
		Error:    err,
	}
	return nil
}

func (h scmpHandler) handle(pkt *snet.Packet) (snet.SCMPEchoReply, error) {
	if pkt.Payload == nil {
		return snet.SCMPEchoReply{}, serrors.New("no v2 payload found")
	}
	r, ok := pkt.Payload.(snet.SCMPEchoReply)
	if !ok {
		return snet.SCMPEchoReply{}, serrors.New("not SCMPEchoReply",
			"type", common.TypeOf(pkt.Payload))
	}
	if r.Identifier != h.id {
		return snet.SCMPEchoReply{}, serrors.New("wrong SCMP ID",
			"expected", h.id, "actual", r.Identifier)
	}
	return r, nil
}
<commit_msg>ping: attach error details if there are interfaces down on the path<commit_after>// Copyright 2020 Anapaya Systems
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package ping implements pinging based on SCMP echo messages.
package ping

import (
	"context"
	"encoding/binary"
	"math/rand"
	"net"
	"time"

	"github.com/scionproto/scion/go/lib/addr"
	"github.com/scionproto/scion/go/lib/common"
	"github.com/scionproto/scion/go/lib/log"
	"github.com/scionproto/scion/go/lib/serrors"
	"github.com/scionproto/scion/go/lib/snet"
	"github.com/scionproto/scion/go/lib/sock/reliable"
	"github.com/scionproto/scion/go/lib/topology/underlay"
)

// Stats contains the statistics of a ping run.
type Stats struct {
	Sent     int
	Received int
}

// Update contains intermediary information about a received echo reply
type Update struct {
	Size     int
	Source   snet.SCIONAddress
	Sequence int
	RTT      time.Duration
	State    State
}

// State indicates the state of the echo reply
type State int

// Possible states.
const (
	Success State = iota
	AfterTimeout
	OutOfOrder
	Duplicate
)

// Config configures the ping run.
type Config struct {
	Dispatcher reliable.Dispatcher
	Local      *snet.UDPAddr
	Remote     *snet.UDPAddr

	// Attempts is the number of pings to send.
	Attempts uint16
	// Interval is the time between sending pings.
	Interval time.Duration
	// Timeout is the time until a ping is considered to have timed out.
	Timeout time.Duration
	// PayloadSize is the size of the SCMP echo payload.
	PayloadSize int

	// ErrHandler is invoked for every error that does not cause pinging to
	// abort. Execution time must be small, as it is run synchronously.
	ErrHandler func(err error)
	// Update handler is invoked for every ping reply. Execution time must be
	// small, as it is run synchronously.
	UpdateHandler func(Update)
}

// Run ping with the configuration. This blocks until the configured number
// attempts is sent, or the context is canceled.
func Run(ctx context.Context, cfg Config) (Stats, error) {
	if cfg.Interval < time.Millisecond {
		return Stats{}, serrors.New("interval below millisecond")
	}

	id := rand.Uint64()
	replies := make(chan reply, 10)

	svc := snet.DefaultPacketDispatcherService{
		Dispatcher: cfg.Dispatcher,
		SCMPHandler: scmpHandler{
			id:      uint16(id),
			replies: replies,
		},
	}
	conn, port, err := svc.Register(ctx, cfg.Local.IA, cfg.Local.Host, addr.SvcNone)
	if err != nil {
		return Stats{}, err
	}

	local := cfg.Local.Copy()
	local.Host.Port = int(port)

	// we need to have at least 8 bytes to store the request time in the
	// payload.
	if cfg.PayloadSize < 8 {
		cfg.PayloadSize = 8
	}
	p := pinger{
		attempts:      cfg.Attempts,
		interval:      cfg.Interval,
		timeout:       cfg.Timeout,
		pldSize:       cfg.PayloadSize,
		pld:           make([]byte, cfg.PayloadSize),
		id:            id,
		conn:          conn,
		local:         local,
		replies:       replies,
		errHandler:    cfg.ErrHandler,
		updateHandler: cfg.UpdateHandler,
	}
	return p.Ping(ctx, cfg.Remote)
}

type pinger struct {
	attempts uint16
	interval time.Duration
	timeout  time.Duration
	pldSize  int

	id      uint64
	conn    snet.PacketConn
	local   *snet.UDPAddr
	replies <-chan reply

	// Handlers
	errHandler    func(error)
	updateHandler func(Update)

	// Mutable state
	pld              []byte
	sentSequence     int
	receivedSequence int
	stats            Stats
}

func (p *pinger) Ping(ctx context.Context, remote *snet.UDPAddr) (Stats, error) {
	p.sentSequence, p.receivedSequence = -1, -1
	send := time.NewTicker(p.interval)
	defer send.Stop()

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	errSend := make(chan error, 1)

	go func() {
		defer log.HandlePanic()
		p.drain(ctx)
	}()

	go func() {
		defer log.HandlePanic()
		for i := uint16(0); i < p.attempts; i++ {
			if err := p.send(remote); err != nil {
				errSend <- serrors.WrapStr("sending", err)
				return
			}
			select {
			case <-send.C:
			case <-ctx.Done():
				return
			}
		}
		time.AfterFunc(p.timeout, cancel)
	}()

	for i := uint16(0); i < p.attempts; i++ {
		select {
		case <-ctx.Done():
			return p.stats, nil
		case err := <-errSend:
			return p.stats, err
		case reply := <-p.replies:
			if reply.Error != nil {
				if p.errHandler != nil {
					p.errHandler(reply.Error)
				}
				continue
			}
			p.receive(reply)
		}
	}
	return p.stats, nil
}

func (p *pinger) send(remote *snet.UDPAddr) error {
	sequence := p.sentSequence + 1

	binary.BigEndian.PutUint64(p.pld, uint64(time.Now().UnixNano()))
	pkt, err := pack(p.local, remote, snet.SCMPEchoRequest{
		Identifier: uint16(p.id),
		SeqNumber:  uint16(sequence),
		Payload:    p.pld,
	})
	if err != nil {
		return err
	}
	nextHop := remote.NextHop
	if nextHop == nil && p.local.IA.Equal(remote.IA) {
		nextHop = &net.UDPAddr{
			IP:   remote.Host.IP,
			Port: underlay.EndhostPort,
			Zone: remote.Host.Zone,
		}

	}
	if err := p.conn.WriteTo(pkt, nextHop); err != nil {
		return err
	}

	p.sentSequence = sequence
	p.stats.Sent++
	return nil
}

func (p *pinger) receive(reply reply) {
	rtt := reply.Received.Sub(time.Unix(0, int64(binary.BigEndian.Uint64(reply.Reply.Payload)))).
		Round(time.Microsecond)
	var state State
	switch {
	case rtt > p.timeout:
		state = AfterTimeout
	case int(reply.Reply.SeqNumber) < p.receivedSequence:
		state = OutOfOrder
	case int(reply.Reply.SeqNumber) == p.receivedSequence:
		state = Duplicate
	default:
		state = Success
		p.receivedSequence = int(reply.Reply.SeqNumber)
	}
	p.stats.Received++
	if p.updateHandler != nil {
		p.updateHandler(Update{
			RTT:      rtt,
			Sequence: int(reply.Reply.SeqNumber),
			Size:     reply.Size,
			Source:   reply.Source,
			State:    state,
		})
	}
}

func (p *pinger) drain(ctx context.Context) {
	var last time.Time
	for {
		select {
		case <-ctx.Done():
			return
		default:
			var pkt snet.Packet
			var ov net.UDPAddr
			if err := p.conn.ReadFrom(&pkt, &ov); err != nil && p.errHandler != nil {
				// Rate limit the error reports.
				if now := time.Now(); now.Sub(last) > 500*time.Millisecond {
					p.errHandler(serrors.WrapStr("reading packet", err))
					last = now
				}
			}
		}
	}
}

type reply struct {
	Received time.Time
	Source   snet.SCIONAddress
	Size     int
	Reply    snet.SCMPEchoReply
	Error    error
}

type scmpHandler struct {
	id      uint16
	replies chan<- reply
}

func (h scmpHandler) Handle(pkt *snet.Packet) error {
	echo, err := h.handle(pkt)
	h.replies <- reply{
		Received: time.Now(),
		Source:   pkt.Source,
		Size:     len(pkt.Bytes),
		Reply:    echo,
		Error:    err,
	}
	return nil
}

func (h scmpHandler) handle(pkt *snet.Packet) (snet.SCMPEchoReply, error) {
	if pkt.Payload == nil {
		return snet.SCMPEchoReply{}, serrors.New("no v2 payload found")
	}
	switch s := pkt.Payload.(type) {
	case snet.SCMPEchoReply:
	case snet.SCMPExternalInterfaceDown:
		return snet.SCMPEchoReply{}, serrors.New("external interface is down",
			"isd_as", s.IA, "interface", s.Interface)
	case snet.SCMPInternalConnectivityDown:
		return snet.SCMPEchoReply{}, serrors.New("internal connectivity is down",
			"isd_as", s.IA, "ingress", s.Ingress, "egress", s.Egress)
	default:
		return snet.SCMPEchoReply{}, serrors.New("not SCMPEchoReply",
			"type", common.TypeOf(pkt.Payload),
		)
	}
	r := pkt.Payload.(snet.SCMPEchoReply)
	if r.Identifier != h.id {
		return snet.SCMPEchoReply{}, serrors.New("wrong SCMP ID",
			"expected", h.id, "actual", r.Identifier)
	}
	return r, nil
}
<|endoftext|>


<commit_before>// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package migrations

import (
	"fmt"

	"code.gitea.io/gitea/models"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/migrations/base"
)

// MigrateOptions is equal to base.MigrateOptions
type MigrateOptions = base.MigrateOptions

var (
	factories []base.DownloaderFactory
)

// RegisterDownloaderFactory registers a downloader factory
func RegisterDownloaderFactory(factory base.DownloaderFactory) {
	factories = append(factories, factory)
}

// MigrateRepository migrate repository according MigrateOptions
func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) {
	var (
		downloader base.Downloader
		uploader   = NewGiteaLocalUploader(doer, ownerName, opts.Name)
	)

	for _, factory := range factories {
		if match, err := factory.Match(opts); err != nil {
			return nil, err
		} else if match {
			downloader, err = factory.New(opts)
			if err != nil {
				return nil, err
			}
			break
		}
	}

	if downloader == nil {
		opts.Wiki = true
		opts.Milestones = false
		opts.Labels = false
		opts.Releases = false
		opts.Comments = false
		opts.Issues = false
		opts.PullRequests = false
		downloader = NewPlainGitDownloader(ownerName, opts.Name, opts.RemoteURL)
		log.Trace("Will migrate from git: %s", opts.RemoteURL)
	}

	if err := migrateRepository(downloader, uploader, opts); err != nil {
		if err1 := uploader.Rollback(); err1 != nil {
			log.Error("rollback failed: %v", err1)
		}
		return nil, err
	}

	return uploader.repo, nil
}

// migrateRepository will download informations and upload to Uploader, this is a simple
// process for small repository. For a big repository, save all the data to disk
// before upload is better
func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions) error {
	repo, err := downloader.GetRepoInfo()
	if err != nil {
		return err
	}
	repo.IsPrivate = opts.Private
	repo.IsMirror = opts.Mirror
	log.Trace("migrating git data")
	if err := uploader.CreateRepo(repo, opts.Wiki); err != nil {
		return err
	}

	if opts.Milestones {
		log.Trace("migrating milestones")
		milestones, err := downloader.GetMilestones()
		if err != nil {
			return err
		}

		for _, milestone := range milestones {
			if err := uploader.CreateMilestone(milestone); err != nil {
				return err
			}
		}
	}

	if opts.Labels {
		log.Trace("migrating labels")
		labels, err := downloader.GetLabels()
		if err != nil {
			return err
		}

		for _, label := range labels {
			if err := uploader.CreateLabel(label); err != nil {
				return err
			}
		}
	}

	if opts.Releases {
		log.Trace("migrating releases")
		releases, err := downloader.GetReleases()
		if err != nil {
			return err
		}

		for _, release := range releases {
			if err := uploader.CreateRelease(release); err != nil {
				return err
			}
		}
	}

	if opts.Issues {
		log.Trace("migrating issues and comments")
		for {
			issues, err := downloader.GetIssues(0, 100)
			if err != nil {
				return err
			}
			for _, issue := range issues {
				if !opts.IgnoreIssueAuthor {
					issue.Content = fmt.Sprintf("Author: @%s \n\n%s", issue.PosterName, issue.Content)
				}

				if err := uploader.CreateIssue(issue); err != nil {
					return err
				}

				if !opts.Comments {
					continue
				}

				comments, err := downloader.GetComments(issue.Number)
				if err != nil {
					return err
				}
				for _, comment := range comments {
					if !opts.IgnoreIssueAuthor {
						comment.Content = fmt.Sprintf("Author: @%s \n\n%s", comment.PosterName, comment.Content)
					}
					if err := uploader.CreateComment(issue.Number, comment); err != nil {
						return err
					}
				}
			}

			if len(issues) < 100 {
				break
			}
		}
	}

	if opts.PullRequests {
		log.Trace("migrating pull requests and comments")
		for {
			prs, err := downloader.GetPullRequests(0, 100)
			if err != nil {
				return err
			}

			for _, pr := range prs {
				if !opts.IgnoreIssueAuthor {
					pr.Content = fmt.Sprintf("Author: @%s \n\n%s", pr.PosterName, pr.Content)
				}
				if err := uploader.CreatePullRequest(pr); err != nil {
					return err
				}
				if !opts.Comments {
					continue
				}

				comments, err := downloader.GetComments(pr.Number)
				if err != nil {
					return err
				}
				for _, comment := range comments {
					if !opts.IgnoreIssueAuthor {
						comment.Content = fmt.Sprintf("Author: @%s \n\n%s", comment.PosterName, comment.Content)
					}
					if err := uploader.CreateComment(pr.Number, comment); err != nil {
						return err
					}
				}
			}
			if len(prs) < 100 {
				break
			}
		}
	}

	return nil
}
<commit_msg>fix missing repo description when migrating (#7000)<commit_after>// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package migrations

import (
	"fmt"

	"code.gitea.io/gitea/models"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/migrations/base"
)

// MigrateOptions is equal to base.MigrateOptions
type MigrateOptions = base.MigrateOptions

var (
	factories []base.DownloaderFactory
)

// RegisterDownloaderFactory registers a downloader factory
func RegisterDownloaderFactory(factory base.DownloaderFactory) {
	factories = append(factories, factory)
}

// MigrateRepository migrate repository according MigrateOptions
func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) {
	var (
		downloader base.Downloader
		uploader   = NewGiteaLocalUploader(doer, ownerName, opts.Name)
	)

	for _, factory := range factories {
		if match, err := factory.Match(opts); err != nil {
			return nil, err
		} else if match {
			downloader, err = factory.New(opts)
			if err != nil {
				return nil, err
			}
			break
		}
	}

	if downloader == nil {
		opts.Wiki = true
		opts.Milestones = false
		opts.Labels = false
		opts.Releases = false
		opts.Comments = false
		opts.Issues = false
		opts.PullRequests = false
		downloader = NewPlainGitDownloader(ownerName, opts.Name, opts.RemoteURL)
		log.Trace("Will migrate from git: %s", opts.RemoteURL)
	}

	if err := migrateRepository(downloader, uploader, opts); err != nil {
		if err1 := uploader.Rollback(); err1 != nil {
			log.Error("rollback failed: %v", err1)
		}
		return nil, err
	}

	return uploader.repo, nil
}

// migrateRepository will download informations and upload to Uploader, this is a simple
// process for small repository. For a big repository, save all the data to disk
// before upload is better
func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions) error {
	repo, err := downloader.GetRepoInfo()
	if err != nil {
		return err
	}
	repo.IsPrivate = opts.Private
	repo.IsMirror = opts.Mirror
	if opts.Description != "" {
		repo.Description = opts.Description
	}
	log.Trace("migrating git data")
	if err := uploader.CreateRepo(repo, opts.Wiki); err != nil {
		return err
	}

	if opts.Milestones {
		log.Trace("migrating milestones")
		milestones, err := downloader.GetMilestones()
		if err != nil {
			return err
		}

		for _, milestone := range milestones {
			if err := uploader.CreateMilestone(milestone); err != nil {
				return err
			}
		}
	}

	if opts.Labels {
		log.Trace("migrating labels")
		labels, err := downloader.GetLabels()
		if err != nil {
			return err
		}

		for _, label := range labels {
			if err := uploader.CreateLabel(label); err != nil {
				return err
			}
		}
	}

	if opts.Releases {
		log.Trace("migrating releases")
		releases, err := downloader.GetReleases()
		if err != nil {
			return err
		}

		for _, release := range releases {
			if err := uploader.CreateRelease(release); err != nil {
				return err
			}
		}
	}

	if opts.Issues {
		log.Trace("migrating issues and comments")
		for {
			issues, err := downloader.GetIssues(0, 100)
			if err != nil {
				return err
			}
			for _, issue := range issues {
				if !opts.IgnoreIssueAuthor {
					issue.Content = fmt.Sprintf("Author: @%s \n\n%s", issue.PosterName, issue.Content)
				}

				if err := uploader.CreateIssue(issue); err != nil {
					return err
				}

				if !opts.Comments {
					continue
				}

				comments, err := downloader.GetComments(issue.Number)
				if err != nil {
					return err
				}
				for _, comment := range comments {
					if !opts.IgnoreIssueAuthor {
						comment.Content = fmt.Sprintf("Author: @%s \n\n%s", comment.PosterName, comment.Content)
					}
					if err := uploader.CreateComment(issue.Number, comment); err != nil {
						return err
					}
				}
			}

			if len(issues) < 100 {
				break
			}
		}
	}

	if opts.PullRequests {
		log.Trace("migrating pull requests and comments")
		for {
			prs, err := downloader.GetPullRequests(0, 100)
			if err != nil {
				return err
			}

			for _, pr := range prs {
				if !opts.IgnoreIssueAuthor {
					pr.Content = fmt.Sprintf("Author: @%s \n\n%s", pr.PosterName, pr.Content)
				}
				if err := uploader.CreatePullRequest(pr); err != nil {
					return err
				}
				if !opts.Comments {
					continue
				}

				comments, err := downloader.GetComments(pr.Number)
				if err != nil {
					return err
				}
				for _, comment := range comments {
					if !opts.IgnoreIssueAuthor {
						comment.Content = fmt.Sprintf("Author: @%s \n\n%s", comment.PosterName, comment.Content)
					}
					if err := uploader.CreateComment(pr.Number, comment); err != nil {
						return err
					}
				}
			}
			if len(prs) < 100 {
				break
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package hostdb

// scan.go contains the functions which periodically scan the list of all hosts
// to see which hosts are online or offline, and to get any updates to the
// settings of the hosts.

import (
	"bytes"
	"crypto/rand"
	"math/big"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

const (
	defaultScanSleep = 1*time.Hour + 37*time.Minute
	maxScanSleep     = 4 * time.Hour
	minScanSleep     = 1 * time.Hour

	maxActiveHosts              = 500
	inactiveHostCheckupQuantity = 250

	maxSettingsLen = 2e3

	hostRequestTimeout = 5 * time.Second

	// scanningThreads is the number of threads that will be probing hosts for
	// their settings and checking for reliability.
	scanningThreads = 25
)

// Reliability is a measure of a host's uptime.
var (
	MaxReliability     = types.NewCurrency64(500) // Given the scanning defaults, about 6 weeks of survival.
	DefaultReliability = types.NewCurrency64(150) // Given the scanning defaults, about 2 week of survival.
	UnreachablePenalty = types.NewCurrency64(1)
)

// addHostToScanPool creates a gofunc that adds a host to the scan pool. If the
// scan pool is currently full, the blocking gofunc will not cause a deadlock.
// The gofunc is created inside of this function to eliminate the burden of
// needing to remember to call 'go addHostToScanPool'.
func (hdb *HostDB) scanHostEntry(entry *hostEntry) {
	go func() {
		hdb.scanPool <- entry
	}()
}

// decrementReliability reduces the reliability of a node, moving it out of the
// set of active hosts or deleting it entirely if necessary.
func (hdb *HostDB) decrementReliability(addr modules.NetAddress, penalty types.Currency) {
	// Look up the entry and decrement the reliability.
	entry, exists := hdb.allHosts[addr]
	if !exists {
		// TODO: should panic here
		return
	}
	entry.Reliability = entry.Reliability.Sub(penalty)
	entry.Online = false

	// If the entry is in the active database, remove it from the active
	// database.
	node, exists := hdb.activeHosts[addr]
	if exists {
		node.removeNode()
		delete(hdb.activeHosts, entry.NetAddress)
	}

	// If the reliability has fallen to 0, remove the host from the
	// database entirely.
	if entry.Reliability.IsZero() {
		delete(hdb.allHosts, addr)
	}
}

// threadedProbeHosts tries to fetch the settings of a host. If successful, the
// host is put in the set of active hosts. If unsuccessful, the host id deleted
// from the set of active hosts.
func (hdb *HostDB) threadedProbeHosts() {
	defer hdb.threadGroup.Done()
	for hostEntry := range hdb.scanPool {
		// Request settings from the queued host entry.
		// TODO: use dialer.Cancel to shutdown quickly
		hdb.log.Debugln("Scanning", hostEntry.NetAddress, hostEntry.PublicKey)
		var settings modules.HostExternalSettings
		err := func() error {
			conn, err := hdb.dialer.DialTimeout(hostEntry.NetAddress, hostRequestTimeout)
			if err != nil {
				return err
			}
			defer conn.Close()
			err = encoding.WriteObject(conn, modules.RPCSettings)
			if err != nil {
				return err
			}
			var pubkey crypto.PublicKey
			copy(pubkey[:], hostEntry.PublicKey.Key)
			return crypto.ReadSignedObject(conn, &settings, maxSettingsLen, pubkey)
		}()
		if err != nil {
			hdb.log.Debugln("Scanning", hostEntry.NetAddress, hostEntry.PublicKey, "failed", err)
		} else {
			hdb.log.Debugln("Scanning", hostEntry.NetAddress, hostEntry.PublicKey, "succeeded")
		}

		// Now that network communication is done, lock the hostdb to modify the
		// host entry.
		func() {
			hdb.mu.Lock()
			defer hdb.mu.Unlock()

			// Regardless of whether the host responded, add it to allHosts.
			priorHost, exists := hdb.allHosts[hostEntry.NetAddress]
			if !exists {
				hdb.allHosts[hostEntry.NetAddress] = hostEntry
			}

			// If the scan was unsuccessful, decrement the host's reliability.
			if err != nil {
				if exists && bytes.Equal(priorHost.PublicKey.Key, hostEntry.PublicKey.Key) {
					// Only decrement the reliability if the public key in the
					// hostdb matches the public key in the host announcement -
					// the failure may just be a failed signature, indicating
					// the wrong public key.
					hdb.decrementReliability(hostEntry.NetAddress, UnreachablePenalty)
				}
				return
			}

			// The hostEntry should be updated to reflect the new weight. The
			// safety properties of the tree require that the weight does not
			// change while the node is in the tree, so the node must be
			// removed before the settings and weight are changed.
			existingNode, exists := hdb.activeHosts[hostEntry.NetAddress]
			if exists {
				existingNode.removeNode()
				delete(hdb.activeHosts, hostEntry.NetAddress)
			}

			// Update the host settings, reliability, and weight. The old NetAddress
			// must be preserved.
			settings.NetAddress = hostEntry.HostExternalSettings.NetAddress
			hostEntry.HostExternalSettings = settings
			hostEntry.Reliability = MaxReliability
			hostEntry.Weight = calculateHostWeight(*hostEntry)
			hostEntry.Online = true

			// If 'maxActiveHosts' has not been reached, add the host to the
			// activeHosts tree.
			if _, exists := hdb.activeHosts[hostEntry.NetAddress]; exists || len(hdb.activeHosts) < maxActiveHosts {
				hdb.insertNode(hostEntry)
			}
			hdb.save()
		}()
	}
}

// threadedScan is an ongoing function which will query the full set of hosts
// every few hours to see who is online and available for uploading.
func (hdb *HostDB) threadedScan() {
	defer hdb.threadGroup.Done()
	for {
		// Determine who to scan. At most 'maxActiveHosts' will be scanned,
		// starting with the active hosts followed by a random selection of the
		// inactive hosts.
		func() {
			hdb.mu.Lock()
			defer hdb.mu.Unlock()

			// Scan all active hosts.
			for _, host := range hdb.activeHosts {
				hdb.scanHostEntry(host.hostEntry)
			}

			// Assemble all of the inactive hosts into a single array.
			var entries []*hostEntry
			for _, entry := range hdb.allHosts {
				_, exists := hdb.activeHosts[entry.NetAddress]
				if !exists {
					entries = append(entries, entry)
				}
			}

			// Generate a random ordering of up to inactiveHostCheckupQuantity
			// hosts.
			hostOrder, err := crypto.Perm(len(entries))
			if err != nil {
				hdb.log.Println("ERR: could not generate random permutation:", err)
			}

			// Scan each host.
			for i := 0; i < len(hostOrder) && i < inactiveHostCheckupQuantity; i++ {
				hdb.scanHostEntry(entries[hostOrder[i]])
			}
		}()

		// Sleep for a random amount of time before doing another round of
		// scanning. The minimums and maximums keep the scan time reasonable,
		// while the randomness prevents the scanning from always happening at
		// the same time of day or week.
		maxBig := big.NewInt(int64(maxScanSleep))
		minBig := big.NewInt(int64(minScanSleep))
		randSleep, err := rand.Int(rand.Reader, maxBig.Sub(maxBig, minBig))
		if err != nil {
			build.Critical(err)
			// If there's an error, sleep for the default amount of time.
			defaultBig := big.NewInt(int64(defaultScanSleep))
			randSleep = defaultBig.Sub(defaultBig, minBig)
		}

		select {
		// awaken and exit if hostdb is closing
		case <-hdb.closeChan:
			return
		case <-time.After(time.Duration(randSleep.Int64()) + minScanSleep):
		}
	}
}
<commit_msg>break the hostEntry update code into a separate function<commit_after>package hostdb

// scan.go contains the functions which periodically scan the list of all hosts
// to see which hosts are online or offline, and to get any updates to the
// settings of the hosts.

import (
	"bytes"
	"crypto/rand"
	"math/big"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

const (
	defaultScanSleep = 1*time.Hour + 37*time.Minute
	maxScanSleep     = 4 * time.Hour
	minScanSleep     = 1 * time.Hour

	maxActiveHosts              = 500
	inactiveHostCheckupQuantity = 250

	maxSettingsLen = 2e3

	hostRequestTimeout = 5 * time.Second

	// scanningThreads is the number of threads that will be probing hosts for
	// their settings and checking for reliability.
	scanningThreads = 25
)

// Reliability is a measure of a host's uptime.
var (
	MaxReliability     = types.NewCurrency64(500) // Given the scanning defaults, about 6 weeks of survival.
	DefaultReliability = types.NewCurrency64(150) // Given the scanning defaults, about 2 week of survival.
	UnreachablePenalty = types.NewCurrency64(1)
)

// addHostToScanPool creates a gofunc that adds a host to the scan pool. If the
// scan pool is currently full, the blocking gofunc will not cause a deadlock.
// The gofunc is created inside of this function to eliminate the burden of
// needing to remember to call 'go addHostToScanPool'.
func (hdb *HostDB) scanHostEntry(entry *hostEntry) {
	go func() {
		hdb.scanPool <- entry
	}()
}

// decrementReliability reduces the reliability of a node, moving it out of the
// set of active hosts or deleting it entirely if necessary.
func (hdb *HostDB) decrementReliability(addr modules.NetAddress, penalty types.Currency) {
	// Look up the entry and decrement the reliability.
	entry, exists := hdb.allHosts[addr]
	if !exists {
		// TODO: should panic here
		return
	}
	entry.Reliability = entry.Reliability.Sub(penalty)
	entry.Online = false

	// If the entry is in the active database, remove it from the active
	// database.
	node, exists := hdb.activeHosts[addr]
	if exists {
		node.removeNode()
		delete(hdb.activeHosts, entry.NetAddress)
	}

	// If the reliability has fallen to 0, remove the host from the
	// database entirely.
	if entry.Reliability.IsZero() {
		delete(hdb.allHosts, addr)
	}
}

// managedUpdateEntry updates an entry in the hostdb after a scan has taken
// place.
func (hdb *HostDB) managedUpdateEntry(hostEntry *hostEntry, settings modules.HostExternalSettings, err error) {
	hdb.mu.Lock()
	defer hdb.mu.Unlock()

	// Regardless of whether the host responded, add it to allHosts.
	priorHost, exists := hdb.allHosts[hostEntry.NetAddress]
	if !exists {
		hdb.allHosts[hostEntry.NetAddress] = hostEntry
	}

	// If the scan was unsuccessful, decrement the host's reliability.
	if err != nil {
		if exists && bytes.Equal(priorHost.PublicKey.Key, hostEntry.PublicKey.Key) {
			// Only decrement the reliability if the public key in the
			// hostdb matches the public key in the host announcement -
			// the failure may just be a failed signature, indicating
			// the wrong public key.
			hdb.decrementReliability(hostEntry.NetAddress, UnreachablePenalty)
		}
		return
	}

	// The hostEntry should be updated to reflect the new weight. The
	// safety properties of the tree require that the weight does not
	// change while the node is in the tree, so the node must be
	// removed before the settings and weight are changed.
	existingNode, exists := hdb.activeHosts[hostEntry.NetAddress]
	if exists {
		existingNode.removeNode()
		delete(hdb.activeHosts, hostEntry.NetAddress)
	}

	// Update the host settings, reliability, and weight. The old NetAddress
	// must be preserved.
	settings.NetAddress = hostEntry.HostExternalSettings.NetAddress
	hostEntry.HostExternalSettings = settings
	hostEntry.Reliability = MaxReliability
	hostEntry.Weight = calculateHostWeight(*hostEntry)
	hostEntry.Online = true

	// If 'maxActiveHosts' has not been reached, add the host to the
	// activeHosts tree.
	if _, exists := hdb.activeHosts[hostEntry.NetAddress]; exists || len(hdb.activeHosts) < maxActiveHosts {
		hdb.insertNode(hostEntry)
	}
	hdb.save()
}

// threadedProbeHosts tries to fetch the settings of a host. If successful, the
// host is put in the set of active hosts. If unsuccessful, the host id deleted
// from the set of active hosts.
func (hdb *HostDB) threadedProbeHosts() {
	defer hdb.threadGroup.Done()
	for hostEntry := range hdb.scanPool {
		// Request settings from the queued host entry.
		// TODO: use dialer.Cancel to shutdown quickly
		hdb.log.Debugln("Scanning", hostEntry.NetAddress, hostEntry.PublicKey)
		var settings modules.HostExternalSettings
		err := func() error {
			conn, err := hdb.dialer.DialTimeout(hostEntry.NetAddress, hostRequestTimeout)
			if err != nil {
				return err
			}
			defer conn.Close()
			err = encoding.WriteObject(conn, modules.RPCSettings)
			if err != nil {
				return err
			}
			var pubkey crypto.PublicKey
			copy(pubkey[:], hostEntry.PublicKey.Key)
			return crypto.ReadSignedObject(conn, &settings, maxSettingsLen, pubkey)
		}()
		if err != nil {
			hdb.log.Debugln("Scanning", hostEntry.NetAddress, hostEntry.PublicKey, "failed", err)
		} else {
			hdb.log.Debugln("Scanning", hostEntry.NetAddress, hostEntry.PublicKey, "succeeded")
		}

		// Update the host tree to have a new entry.
		hdb.managedUpdateEntry(hostEntry, settings, err)
	}
}

// threadedScan is an ongoing function which will query the full set of hosts
// every few hours to see who is online and available for uploading.
func (hdb *HostDB) threadedScan() {
	defer hdb.threadGroup.Done()
	for {
		// Determine who to scan. At most 'maxActiveHosts' will be scanned,
		// starting with the active hosts followed by a random selection of the
		// inactive hosts.
		func() {
			hdb.mu.Lock()
			defer hdb.mu.Unlock()

			// Scan all active hosts.
			for _, host := range hdb.activeHosts {
				hdb.scanHostEntry(host.hostEntry)
			}

			// Assemble all of the inactive hosts into a single array.
			var entries []*hostEntry
			for _, entry := range hdb.allHosts {
				_, exists := hdb.activeHosts[entry.NetAddress]
				if !exists {
					entries = append(entries, entry)
				}
			}

			// Generate a random ordering of up to inactiveHostCheckupQuantity
			// hosts.
			hostOrder, err := crypto.Perm(len(entries))
			if err != nil {
				hdb.log.Println("ERR: could not generate random permutation:", err)
			}

			// Scan each host.
			for i := 0; i < len(hostOrder) && i < inactiveHostCheckupQuantity; i++ {
				hdb.scanHostEntry(entries[hostOrder[i]])
			}
		}()

		// Sleep for a random amount of time before doing another round of
		// scanning. The minimums and maximums keep the scan time reasonable,
		// while the randomness prevents the scanning from always happening at
		// the same time of day or week.
		maxBig := big.NewInt(int64(maxScanSleep))
		minBig := big.NewInt(int64(minScanSleep))
		randSleep, err := rand.Int(rand.Reader, maxBig.Sub(maxBig, minBig))
		if err != nil {
			build.Critical(err)
			// If there's an error, sleep for the default amount of time.
			defaultBig := big.NewInt(int64(defaultScanSleep))
			randSleep = defaultBig.Sub(defaultBig, minBig)
		}

		select {
		// awaken and exit if hostdb is closing
		case <-hdb.closeChan:
			return
		case <-time.After(time.Duration(randSleep.Int64()) + minScanSleep):
		}
	}
}
<|endoftext|>


<commit_before>package page_runner

import (
	"fmt"
	"github.com/jesusslim/slimmysql"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"
)

type PageRunner struct {
	id        string
	waitGroup sync.WaitGroup
	sql       *slimmysql.Sql
	path      string    //扫描路径
	suffix    string    //文件后缀
	baseUrl   string    //http基础url
	ignore    []string  //忽略的目录/文件名
	rep       []string  //需替换为空的字符串
	extError  []string  //其他错误标示
	c         chan bool //channel 建议根据服务器情况调整合适大小
	result    map[string]map[string]interface{}
	cookie    string //cookie for sessionid
	maxTimes  int    //最大尝试访问次数
}

func NewPageRunner(id string, path string, suffix string, baseUrl string, ignore []string, rep []string, extErr []string, sql *slimmysql.Sql, channelLength int, cookie string, maxTimes int) *PageRunner {
	if !strings.HasSuffix(baseUrl, "/") {
		baseUrl = baseUrl + "/"
	}
	if suffix == "" {
		suffix = "index.html"
	}
	if channelLength == 0 {
		channelLength = 1
	}
	if maxTimes == 0 {
		maxTimes = 10
	}
	return &PageRunner{
		sql:      sql,
		path:     path,
		suffix:   suffix,
		baseUrl:  baseUrl,
		ignore:   ignore,
		extError: extErr,
		rep:      rep,
		c:        make(chan bool, channelLength),
		result:   make(map[string]map[string]interface{}),
		cookie:   cookie,
		maxTimes: maxTimes,
	}
}

//for thinkphp
func NewPageRunnerTP(id string, path, baseUrl string, rp []string, extErr []string, sql *slimmysql.Sql, channelLength int, cookie string, maxTimes int) *PageRunner {
	return NewPageRunner(
		id,
		path,
		".html",
		baseUrl,
		[]string{
			"Widget",
		},
		append(
			[]string{
				"View/",
				"View/index.html",
			}, rp),
		extErr,
		sql,
		channelLength,
		cookie,
		maxTimes)
}

type UrlModel struct {
	module     string
	controller string
	action     string
	url        string
	times      int
	lastErr    string
}

func NewUrlModel(baseUrl, subUrl string) *UrlModel {
	mdl := &UrlModel{
		url:   baseUrl + subUrl,
		times: 0,
	}
	subs := strings.Split(subUrl, "/")
	l := len(subs)
	if l > 0 {
		mdl.module = subs[0]
	}
	if l > 1 {
		mdl.controller = subs[1]
	}
	if l > 2 {
		mdl.action = subs[2]
	}
	return mdl
}

func (this *PageRunner) fetchUrl(id string, url *UrlModel) {
	this.c <- true
	start := time.Now()
	var err error
	var resp *http.Response
	if this.cookie == "" {
		//1.easy
		resp, err = http.Get(url.url)
	} else {
		//2.with cookie
		client := &http.Client{}
		req, _ := http.NewRequest("GET", url.url, nil)
		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
		req.Header.Set("Cookie", this.cookie)
		resp, err = client.Do(req)
	}
	if err != nil {
		<-this.c
		fmt.Println("ERROR:" + err.Error())
		if url.times == this.maxTimes-1 {
			//结束 give up
			this.result[id] = map[string]interface{}{
				"duration":    0,
				"url":         url.url,
				"status":      -1,
				"create_time": time.Now().Unix(),
				"same":        0,
				"task_id":     this.id,
				"is_err":      1,
				"err":         url.lastErr,
				"module":      url.module,
				"controller":  url.controller,
				"action":      url.action,
				"times":       url.times + 1,
			}
			this.waitGroup.Done()
		} else {
			url.times++
			url.lastErr = err.Error()
			this.fetchUrl(id, url)
		}
	} else {
		duration := time.Since(start).Seconds() * 1000
		defer resp.Body.Close()

		_, ok := this.result[id]

		if !ok {
			status := resp.StatusCode
			create_time := time.Now().Unix()
			header := resp.Header
			last_url := resp.Request.URL.String()
			same := 0
			is_err := 0
			if status != 200 {
				is_err = 1
			}
			if strings.EqualFold(url.url, last_url) {
				same = 1
			}
			// fmt.Println(header.Get("Date"))
			server := header.Get("Server")
			//xby := header.Get("X-Powered-By")
			if same == 0 {
				for _, e := range this.extError {
					if strings.Contains(last_url, e) {
						is_err = 1
						break
					}
				}
			}

			this.result[id] = map[string]interface{}{
				"duration":    int(duration),
				"url":         url.url,
				"status":      status,
				"create_time": create_time,
				"server":      server,
				"last_url":    last_url,
				"same":        same,
				"task_id":     this.id,
				"is_err":      is_err,
				"module":      url.module,
				"controller":  url.controller,
				"action":      url.action,
				"err":         url.lastErr,
				"times":       url.times + 1,
			}
		}

		<-this.c
		this.waitGroup.Done()
	}
}

func (this *PageRunner) walkDir() (map[string]*UrlModel, error) {
	files := map[string]*UrlModel{}
	suffix := strings.ToUpper(this.suffix)
	id := 0
	err := filepath.Walk(this.path, func(filename string, info os.FileInfo, err_inside error) error {
		if info.IsDir() {
			return nil
		}
		if strings.HasSuffix(strings.ToUpper(info.Name()), suffix) {
			for _, ig := range this.ignore {
				if strings.Contains(filename, ig) {
					return nil
				}
			}
			for _, rp := range this.rep {
				filename = strings.Replace(filename, rp, "", -1)
			}
			if strings.HasPrefix(filename, "/") {
				filename = filename[1:]
			}
			files[strconv.Itoa(id)] = NewUrlModel(this.baseUrl, filename)
			id++
		}
		return nil
	})
	return files, err
}

func (this *PageRunner) Run() {
	urls, err := this.walkDir()
	if err != nil {
		fmt.Println("ERROR:", err.Error())
	} else {
		num := len(urls)
		id, _ := this.sql.Table("task").Add(map[string]interface{}{
			"num":         num,
			"title":       "Test",
			"create_time": time.Now().Unix(),
		})
		this.id = string(strconv.FormatInt(id, 10))
		for k, v := range urls {
			this.waitGroup.Add(1)
			go this.fetchUrl(k, v)
		}
		this.waitGroup.Wait()
		fmt.Println("DataOk")
		success := len(this.result)
		this.sql.Clear().Table("task").Where("id = "+this.id).SetInc("success", success)
		this.sql = this.sql.Clear()
		for _, v := range this.result {
			this.sql.Table("url").Add(v)
		}
		fmt.Println("Finished")

	}
}

//example
// func main() {
// 	slimmysql.RegisterConnectionDefault(false, "127.0.0.1", "3307", "test", "root", "root", "", false)
// 	sql, _ := slimmysql.NewSqlInstanceDefault()
// 	baseUrl := "http://localhost:8888/teenager/Student"
// 	path := "/Applications/MAMP/htdocs/teenager/Application/Student"
// 	extErr := []string{
// 		"404",
// 		"error",
// 	}
// 	runner := NewPageRunnerTP("0", path, baseUrl,[]string{"/Applications/MAMP/htdocs/teenager/Application/"}, extErr, sql, 30, "PHPSESSID=c3146dcc95ba4e5992441718296aef1d", 50)
// 	runner.Run()
// }
<commit_msg>重访问次数<commit_after>package page_runner

import (
	"fmt"
	"github.com/jesusslim/slimmysql"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"
)

type PageRunner struct {
	id        string
	waitGroup sync.WaitGroup
	sql       *slimmysql.Sql
	path      string    //扫描路径
	suffix    string    //文件后缀
	baseUrl   string    //http基础url
	ignore    []string  //忽略的目录/文件名
	rep       []string  //需替换为空的字符串
	extError  []string  //其他错误标示
	c         chan bool //channel 建议根据服务器情况调整合适大小
	result    map[string]map[string]interface{}
	cookie    string //cookie for sessionid
	maxTimes  int    //最大尝试访问次数
}

func NewPageRunner(id string, path string, suffix string, baseUrl string, ignore []string, rep []string, extErr []string, sql *slimmysql.Sql, channelLength int, cookie string, maxTimes int) *PageRunner {
	if !strings.HasSuffix(baseUrl, "/") {
		baseUrl = baseUrl + "/"
	}
	if suffix == "" {
		suffix = "index.html"
	}
	if channelLength == 0 {
		channelLength = 1
	}
	if maxTimes == 0 {
		maxTimes = 20
	}
	return &PageRunner{
		sql:      sql,
		path:     path,
		suffix:   suffix,
		baseUrl:  baseUrl,
		ignore:   ignore,
		extError: extErr,
		rep:      rep,
		c:        make(chan bool, channelLength),
		result:   make(map[string]map[string]interface{}),
		cookie:   cookie,
		maxTimes: maxTimes,
	}
}

//for thinkphp
func NewPageRunnerTP(id string, path, baseUrl string, rp []string, extErr []string, sql *slimmysql.Sql, channelLength int, cookie string, maxTimes int) *PageRunner {
	return NewPageRunner(
		id,
		path,
		".html",
		baseUrl,
		[]string{
			"Widget",
		},
		append(
			[]string{
				"View/",
				"View/index.html",
			}, rp),
		extErr,
		sql,
		channelLength,
		cookie,
		maxTimes)
}

type UrlModel struct {
	module     string
	controller string
	action     string
	url        string
	times      int
	lastErr    string
}

func NewUrlModel(baseUrl, subUrl string) *UrlModel {
	mdl := &UrlModel{
		url:   baseUrl + subUrl,
		times: 0,
	}
	subs := strings.Split(subUrl, "/")
	l := len(subs)
	if l > 0 {
		mdl.module = subs[0]
	}
	if l > 1 {
		mdl.controller = subs[1]
	}
	if l > 2 {
		mdl.action = subs[2]
	}
	return mdl
}

func (this *PageRunner) fetchUrl(id string, url *UrlModel) {
	this.c <- true
	start := time.Now()
	var err error
	var resp *http.Response
	if this.cookie == "" {
		//1.easy
		resp, err = http.Get(url.url)
	} else {
		//2.with cookie
		client := &http.Client{}
		req, _ := http.NewRequest("GET", url.url, nil)
		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
		req.Header.Set("Cookie", this.cookie)
		resp, err = client.Do(req)
	}
	if err != nil {
		<-this.c
		fmt.Println("ERROR:" + err.Error())
		if url.times == this.maxTimes-1 {
			//结束 give up
			this.result[id] = map[string]interface{}{
				"duration":    0,
				"url":         url.url,
				"status":      -1,
				"create_time": time.Now().Unix(),
				"same":        0,
				"task_id":     this.id,
				"is_err":      1,
				"err":         url.lastErr,
				"module":      url.module,
				"controller":  url.controller,
				"action":      url.action,
				"times":       url.times + 1,
			}
			this.waitGroup.Done()
		} else {
			url.times++
			url.lastErr = err.Error()
			this.fetchUrl(id, url)
		}
	} else {
		duration := time.Since(start).Seconds() * 1000
		defer resp.Body.Close()

		_, ok := this.result[id]

		if !ok {
			status := resp.StatusCode
			create_time := time.Now().Unix()
			header := resp.Header
			last_url := resp.Request.URL.String()
			same := 0
			is_err := 0
			if status != 200 {
				is_err = 1
			}
			if strings.EqualFold(url.url, last_url) {
				same = 1
			}
			// fmt.Println(header.Get("Date"))
			server := header.Get("Server")
			//xby := header.Get("X-Powered-By")
			if same == 0 {
				for _, e := range this.extError {
					if strings.Contains(last_url, e) {
						is_err = 1
						break
					}
				}
			}

			this.result[id] = map[string]interface{}{
				"duration":    int(duration),
				"url":         url.url,
				"status":      status,
				"create_time": create_time,
				"server":      server,
				"last_url":    last_url,
				"same":        same,
				"task_id":     this.id,
				"is_err":      is_err,
				"module":      url.module,
				"controller":  url.controller,
				"action":      url.action,
				"err":         url.lastErr,
				"times":       url.times + 1,
			}
		}

		<-this.c
		this.waitGroup.Done()
	}
}

func (this *PageRunner) walkDir() (map[string]*UrlModel, error) {
	files := map[string]*UrlModel{}
	suffix := strings.ToUpper(this.suffix)
	id := 0
	err := filepath.Walk(this.path, func(filename string, info os.FileInfo, err_inside error) error {
		if info.IsDir() {
			return nil
		}
		if strings.HasSuffix(strings.ToUpper(info.Name()), suffix) {
			for _, ig := range this.ignore {
				if strings.Contains(filename, ig) {
					return nil
				}
			}
			for _, rp := range this.rep {
				filename = strings.Replace(filename, rp, "", -1)
			}
			if strings.HasPrefix(filename, "/") {
				filename = filename[1:]
			}
			files[strconv.Itoa(id)] = NewUrlModel(this.baseUrl, filename)
			id++
		}
		return nil
	})
	return files, err
}

func (this *PageRunner) Run() {
	urls, err := this.walkDir()
	if err != nil {
		fmt.Println("ERROR:", err.Error())
	} else {
		num := len(urls)
		id, _ := this.sql.Table("task").Add(map[string]interface{}{
			"num":         num,
			"title":       "Test",
			"create_time": time.Now().Unix(),
		})
		this.id = string(strconv.FormatInt(id, 10))
		for k, v := range urls {
			this.waitGroup.Add(1)
			go this.fetchUrl(k, v)
		}
		this.waitGroup.Wait()
		fmt.Println("DataOk")
		success := len(this.result)
		this.sql.Clear().Table("task").Where("id = "+this.id).SetInc("success", success)
		this.sql = this.sql.Clear()
		for _, v := range this.result {
			this.sql.Table("url").Add(v)
		}
		fmt.Println("Finished")

	}
}

//example
// func main() {
// 	slimmysql.RegisterConnectionDefault(false, "127.0.0.1", "3307", "test", "root", "root", "", false)
// 	sql, _ := slimmysql.NewSqlInstanceDefault()
// 	baseUrl := "http://localhost:8888/teenager/Student"
// 	path := "/Applications/MAMP/htdocs/teenager/Application/Student"
// 	extErr := []string{
// 		"404",
// 		"error",
// 	}
// 	runner := NewPageRunnerTP("0", path, baseUrl,[]string{"/Applications/MAMP/htdocs/teenager/Application/"}, extErr, sql, 30, "PHPSESSID=c3146dcc95ba4e5992441718296aef1d", 50)
// 	runner.Run()
// }
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"github.com/dynport/gocloud/aws/s3"
	"io"
	"io/ioutil"
	"net/http"
	"path"
	"strconv"
	"strings"
)

type S3Resource struct {
	Client  *s3.Client
	Bucket  string
	Request *http.Request
}

func (resource *S3Resource) Open() (io.Reader, int64, error) {
	rsp, e := resource.Client.Get(resource.Bucket, resource.key())
	if e != nil {
		return nil, 0, e
	}
	if rsp.Status[0] != '2' {
		return nil, 0, wrongStatus("2xx", rsp.Status)
	}
	i, e := strconv.ParseInt(rsp.Header.Get("Content-Length"), 10, 64)
	if e != nil {
		return nil, 0, e
	}
	return rsp.Body, i, nil
}

func wrongStatus(expected string, found string) error {
	return fmt.Errorf("expected status of %s but got %s", expected, found)
}

func (resource *S3Resource) DockerSize() (int64, error) {
	rsp, e := resource.Client.Head(resource.Bucket, path.Dir(resource.key())+"/layer")
	if e != nil {
		return 0, e
	}
	if rsp.Status[0] != '2' {
		return 0, fmt.Errorf("expected status of 2xx but got", rsp.Status)
	}
	return strconv.ParseInt(rsp.Header.Get("Content-Length"), 10, 64)
}

func (resource *S3Resource) Exists() bool {
	rsp, e := resource.Client.Head(resource.Bucket, resource.key())
	if e != nil {
		return false
	}
	defer rsp.Body.Close()
	return rsp.Status[0] == '2'
}

func (resource *S3Resource) Tags() (map[string]string, error) {
	res, e := resource.Client.ListBucketWithOptions(resource.Bucket, &s3.ListBucketOptions{
		Prefix: resource.key(),
	})
	if e != nil {
		return nil, e
	}
	m := map[string]string{}
	for _, key := range res.Contents {
		rsp, e := resource.Client.Get(resource.Bucket, key.Key)
		if e != nil {
			return nil, e
		}
		if rsp.Status[0] != '2' {
			return nil, fmt.Errorf("expected status 2xx but got %s", rsp.Status)
		}
		b, e := ioutil.ReadAll(rsp.Body)
		if e != nil {
			return nil, e
		}
		m[path.Base(key.Key)] = strings.Replace(string(b), `"`, "", -1)
	}
	return m, nil
}

func (r *S3Resource) LoadResource(p string) ([]byte, error) {
	key := strings.TrimPrefix(p, "/")
	rsp, e := r.Client.Get(r.Bucket, key)
	if e != nil {
		return nil, e
	}
	if rsp.Status[0] != '2' {
		return nil, fmt.Errorf("expected status 2xx but got %s", rsp.Status)
	}
	return ioutil.ReadAll(rsp.Body)
}

func (resource *S3Resource) Store() error {
	opts := &s3.PutOptions{
		ServerSideEncryption: true,
		ContentType:          resource.Request.Header.Get("Content-Type"),
	}
	buf := bytes.NewBuffer(make([]byte, 0, s3.MinPartSize))
	_, e := io.CopyN(buf, resource.Request.Body, s3.MinPartSize)
	if e == io.EOF {
		// less than min multipart size => direct upload
		return resource.Client.Put(resource.Bucket, resource.key(), buf.Bytes(), opts)
	} else if e != nil {
		return e
	}
	mr := io.MultiReader(buf, resource.Request.Body)

	mo := &s3.MultipartOptions{
		PartSize: 5 * 1024 * 1024,
		Callback: func(res *s3.UploadPartResult) {
			if res.Error != nil {
				logger.Print("ERROR: " + e.Error())
			} else if res.Part != nil {
				logger.Printf("uploaded: %03d (%s) %d", res.Part.PartNumber, res.Part.ETag, res.CurrentSize)
			}
		},
		PutOptions: opts,
	}
	_, e = resource.Client.PutMultipart(resource.Bucket, resource.key(), mr, mo)
	return e
}

func (resource *S3Resource) key() string {
	return strings.TrimPrefix(resource.Request.URL.String(), "/")
}
<commit_msg>fixed go vet finding<commit_after>package main

import (
	"bytes"
	"fmt"
	"github.com/dynport/gocloud/aws/s3"
	"io"
	"io/ioutil"
	"net/http"
	"path"
	"strconv"
	"strings"
)

type S3Resource struct {
	Client  *s3.Client
	Bucket  string
	Request *http.Request
}

func (resource *S3Resource) Open() (io.Reader, int64, error) {
	rsp, e := resource.Client.Get(resource.Bucket, resource.key())
	if e != nil {
		return nil, 0, e
	}
	if rsp.Status[0] != '2' {
		return nil, 0, wrongStatus("2xx", rsp.Status)
	}
	i, e := strconv.ParseInt(rsp.Header.Get("Content-Length"), 10, 64)
	if e != nil {
		return nil, 0, e
	}
	return rsp.Body, i, nil
}

func wrongStatus(expected string, found string) error {
	return fmt.Errorf("expected status of %s but got %s", expected, found)
}

func (resource *S3Resource) DockerSize() (int64, error) {
	rsp, e := resource.Client.Head(resource.Bucket, path.Dir(resource.key())+"/layer")
	if e != nil {
		return 0, e
	}
	if rsp.Status[0] != '2' {
		return 0, fmt.Errorf("expected status of 2xx but got %q", rsp.Status)
	}
	return strconv.ParseInt(rsp.Header.Get("Content-Length"), 10, 64)
}

func (resource *S3Resource) Exists() bool {
	rsp, e := resource.Client.Head(resource.Bucket, resource.key())
	if e != nil {
		return false
	}
	defer rsp.Body.Close()
	return rsp.Status[0] == '2'
}

func (resource *S3Resource) Tags() (map[string]string, error) {
	res, e := resource.Client.ListBucketWithOptions(resource.Bucket, &s3.ListBucketOptions{
		Prefix: resource.key(),
	})
	if e != nil {
		return nil, e
	}
	m := map[string]string{}
	for _, key := range res.Contents {
		rsp, e := resource.Client.Get(resource.Bucket, key.Key)
		if e != nil {
			return nil, e
		}
		if rsp.Status[0] != '2' {
			return nil, fmt.Errorf("expected status 2xx but got %s", rsp.Status)
		}
		b, e := ioutil.ReadAll(rsp.Body)
		if e != nil {
			return nil, e
		}
		m[path.Base(key.Key)] = strings.Replace(string(b), `"`, "", -1)
	}
	return m, nil
}

func (r *S3Resource) LoadResource(p string) ([]byte, error) {
	key := strings.TrimPrefix(p, "/")
	rsp, e := r.Client.Get(r.Bucket, key)
	if e != nil {
		return nil, e
	}
	if rsp.Status[0] != '2' {
		return nil, fmt.Errorf("expected status 2xx but got %s", rsp.Status)
	}
	return ioutil.ReadAll(rsp.Body)
}

func (resource *S3Resource) Store() error {
	opts := &s3.PutOptions{
		ServerSideEncryption: true,
		ContentType:          resource.Request.Header.Get("Content-Type"),
	}
	buf := bytes.NewBuffer(make([]byte, 0, s3.MinPartSize))
	_, e := io.CopyN(buf, resource.Request.Body, s3.MinPartSize)
	if e == io.EOF {
		// less than min multipart size => direct upload
		return resource.Client.Put(resource.Bucket, resource.key(), buf.Bytes(), opts)
	} else if e != nil {
		return e
	}
	mr := io.MultiReader(buf, resource.Request.Body)

	mo := &s3.MultipartOptions{
		PartSize: 5 * 1024 * 1024,
		Callback: func(res *s3.UploadPartResult) {
			if res.Error != nil {
				logger.Print("ERROR: " + e.Error())
			} else if res.Part != nil {
				logger.Printf("uploaded: %03d (%s) %d", res.Part.PartNumber, res.Part.ETag, res.CurrentSize)
			}
		},
		PutOptions: opts,
	}
	_, e = resource.Client.PutMultipart(resource.Bucket, resource.key(), mr, mo)
	return e
}

func (resource *S3Resource) key() string {
	return strings.TrimPrefix(resource.Request.URL.String(), "/")
}
<|endoftext|>