<commit_before>package main

import (
	"context"
	log "github.com/Sirupsen/logrus"
	"github.com/docker/docker/api/types"
	docker "github.com/docker/docker/client"
	"github.com/stefanprodan/syros/models"
	"strings"
	"time"
)

type DockerCollector struct {
	ApiAddress  string
	Environment string
	Topic       string
	Client      *docker.Client
	StopChan    chan bool
}

func NewDockerCollector(address string, env string) (*DockerCollector, error) {

	defaultHeaders := map[string]string{"User-Agent": "engine-api-cli-1.0"}
	client, err := docker.NewClient(address, "", nil, defaultHeaders)
	if err != nil {
		return nil, err
	}
	collector := &DockerCollector{
		ApiAddress:  address,
		Environment: env,
		Topic:       "docker",
		Client:      client,
		StopChan:    make(chan bool, 1),
	}

	return collector, nil
}

func (col *DockerCollector) Collect() (*models.DockerPayload, error) {
	start := time.Now().UTC()
	payload := &models.DockerPayload{}

	host, err := col.Client.Info(context.Background())
	if err != nil {
		return nil, err
	}
	payload.Host = MapDockerHost(col.Environment, host)

	options := types.ContainerListOptions{All: true}
	containers, err := col.Client.ContainerList(context.Background(), options)
	if err != nil {
		return nil, err
	}

	payload.Containers = make([]models.DockerContainer, 0)

	for _, container := range containers {
		containerInfo, err := col.Client.ContainerInspect(context.Background(), container.ID)
		if err != nil {
			log.Error(err)
			continue
		}
		payload.Containers = append(payload.Containers, MapDockerContainer(col.Environment, host.ID, host.Name, container, containerInfo))
	}

	log.Debugf("%v collect duration: %v containers %v", col.ApiAddress, time.Now().UTC().Sub(start), len(payload.Containers))
	return payload, nil
}

func MapDockerHost(environment string, info types.Info) models.DockerHost {
	host := models.DockerHost{
		Id:                 models.Hash(info.Name),
		Containers:         info.Containers,
		ContainersRunning:  info.ContainersRunning,
		ContainersPaused:   info.ContainersPaused,
		ContainersStopped:  info.ContainersStopped,
		Images:             info.Images,
		Driver:             info.Driver,
		SystemTime:         info.SystemTime,
		LoggingDriver:      info.LoggingDriver,
		CgroupDriver:       info.CgroupDriver,
		NEventsListener:    info.NEventsListener,
		KernelVersion:      info.KernelVersion,
		OperatingSystem:    info.OperatingSystem,
		OSType:             info.OSType,
		Architecture:       info.Architecture,
		IndexServerAddress: info.IndexServerAddress,
		NCPU:               info.NCPU,
		MemTotal:           info.MemTotal,
		DockerRootDir:      info.DockerRootDir,
		HTTPProxy:          info.HTTPProxy,
		HTTPSProxy:         info.HTTPSProxy,
		NoProxy:            info.NoProxy,
		Name:               info.Name,
		Labels:             info.Labels,
		ExperimentalBuild:  info.ExperimentalBuild,
		ServerVersion:      info.ServerVersion,
		ClusterStore:       info.ClusterStore,
		ClusterAdvertise:   info.ClusterAdvertise,
		DefaultRuntime:     info.DefaultRuntime,
		LiveRestoreEnabled: info.LiveRestoreEnabled,
		Collected:          time.Now().UTC(),
		Environment:        environment,
	}

	for _, reg := range info.RegistryConfig.IndexConfigs {
		host.Registries = append(host.Registries, reg.Name)
	}

	return host
}

func MapDockerContainer(environment string, hostId string, hostName string, c types.Container, cj types.ContainerJSON) models.DockerContainer {
	container := models.DockerContainer{
		Id:           c.ID,
		HostId:       models.Hash(hostName),
		HostName:     hostName,
		Image:        c.Image,
		Command:      c.Command,
		State:        c.State,
		Status:       c.Status,
		Path:         cj.ContainerJSONBase.Path,
		Args:         cj.ContainerJSONBase.Args,
		Name:         cj.ContainerJSONBase.Name,
		RestartCount: cj.ContainerJSONBase.RestartCount,
		PortBindings: make(map[string]string),
		Collected:    time.Now().UTC(),
		Environment:  environment,
	}

	container.Labels = make(map[string]string)
	for key, value := range c.Labels {
		k := strings.Replace(key, ".", "_", -1)
		container.Labels[k] = value
	}

	container.Created, _ = time.Parse(time.RFC3339, cj.ContainerJSONBase.Created)
	if len(container.Name) > 1 {
		container.Name = container.Name[1:len(container.Name)]
	}

	if cj.Config != nil {
		container.Env = cj.Config.Env
	}

	if cj.ContainerJSONBase.State != nil {

		container.StartedAt, _ = time.Parse(time.RFC3339, cj.ContainerJSONBase.State.StartedAt)
		container.FinishedAt, _ = time.Parse(time.RFC3339, cj.ContainerJSONBase.State.FinishedAt)
		container.ExitCode = cj.ContainerJSONBase.State.ExitCode
		container.Error = cj.ContainerJSONBase.State.Error
	}

	if cj.ContainerJSONBase.HostConfig != nil {
		container.NetworkMode = string(cj.ContainerJSONBase.HostConfig.NetworkMode)
		container.RestartPolicy = cj.ContainerJSONBase.HostConfig.RestartPolicy.Name
		for key, val := range cj.ContainerJSONBase.HostConfig.PortBindings {
			if len(val) > 0 {
				container.PortBindings[string(key)] = val[0].HostPort
			}
		}
	}

	// use first Host port bind as container Port
	// if multiple ports are present search for gliderlabs/registrator meta
	container.Port = GetPortFromEnv(container.PortBindings, container.Env)

	return container
}

func GetPortFromEnv(portBindings map[string]string, env []string) string {
	port := ""
	if len(portBindings) == 0 {
		return port
	} else if len(portBindings) == 1 {
		for _, val := range portBindings {
			if len(val) > 0 {
				return val
			}
		}
	}
	if len(env) == 0 {
		for _, val := range portBindings {
			if len(val) > 0 {
				return val
			}
		}
	}
	for _, val := range portBindings {
		if len(val) > 0 {
			registratorMeta := "SERVICE_" + val + "_NAME"
			for _, e := range env {
				if strings.Contains(e, registratorMeta) {
					return val
				}
			}
		}
	}
	if port == "" {
		for _, val := range portBindings {
			if len(val) > 0 {
				return val
			}
		}
	}

	return port
}
<commit_msg>close docker client after each collect<commit_after>package main

import (
	"context"
	log "github.com/Sirupsen/logrus"
	"github.com/docker/docker/api/types"
	docker "github.com/docker/docker/client"
	"github.com/stefanprodan/syros/models"
	"strings"
	"time"
)

type DockerCollector struct {
	ApiAddress  string
	Environment string
	Topic       string
	StopChan    chan bool
}

func NewDockerCollector(address string, env string) (*DockerCollector, error) {

	collector := &DockerCollector{
		ApiAddress:  address,
		Environment: env,
		Topic:       "docker",
		StopChan:    make(chan bool, 1),
	}

	return collector, nil
}

func (col *DockerCollector) Collect() (*models.DockerPayload, error) {
	start := time.Now().UTC()
	payload := &models.DockerPayload{}
	defaultHeaders := map[string]string{"User-Agent": "engine-api-cli-1.0"}
	client, err := docker.NewClient(col.ApiAddress, "", nil, defaultHeaders)
	defer client.Close()
	if err != nil {
		return nil, err
	}
	host, err := client.Info(context.Background())
	if err != nil {
		return nil, err
	}
	payload.Host = MapDockerHost(col.Environment, host)

	options := types.ContainerListOptions{All: true}
	containers, err := client.ContainerList(context.Background(), options)
	if err != nil {
		return nil, err
	}

	payload.Containers = make([]models.DockerContainer, 0)

	for _, container := range containers {
		containerInfo, err := client.ContainerInspect(context.Background(), container.ID)
		if err != nil {
			log.Error(err)
			continue
		}
		payload.Containers = append(payload.Containers, MapDockerContainer(col.Environment, host.ID, host.Name, container, containerInfo))
	}

	log.Debugf("%v collect duration: %v containers %v", col.ApiAddress, time.Now().UTC().Sub(start), len(payload.Containers))
	return payload, nil
}

func MapDockerHost(environment string, info types.Info) models.DockerHost {
	host := models.DockerHost{
		Id:                 models.Hash(info.Name),
		Containers:         info.Containers,
		ContainersRunning:  info.ContainersRunning,
		ContainersPaused:   info.ContainersPaused,
		ContainersStopped:  info.ContainersStopped,
		Images:             info.Images,
		Driver:             info.Driver,
		SystemTime:         info.SystemTime,
		LoggingDriver:      info.LoggingDriver,
		CgroupDriver:       info.CgroupDriver,
		NEventsListener:    info.NEventsListener,
		KernelVersion:      info.KernelVersion,
		OperatingSystem:    info.OperatingSystem,
		OSType:             info.OSType,
		Architecture:       info.Architecture,
		IndexServerAddress: info.IndexServerAddress,
		NCPU:               info.NCPU,
		MemTotal:           info.MemTotal,
		DockerRootDir:      info.DockerRootDir,
		HTTPProxy:          info.HTTPProxy,
		HTTPSProxy:         info.HTTPSProxy,
		NoProxy:            info.NoProxy,
		Name:               info.Name,
		Labels:             info.Labels,
		ExperimentalBuild:  info.ExperimentalBuild,
		ServerVersion:      info.ServerVersion,
		ClusterStore:       info.ClusterStore,
		ClusterAdvertise:   info.ClusterAdvertise,
		DefaultRuntime:     info.DefaultRuntime,
		LiveRestoreEnabled: info.LiveRestoreEnabled,
		Collected:          time.Now().UTC(),
		Environment:        environment,
	}

	for _, reg := range info.RegistryConfig.IndexConfigs {
		host.Registries = append(host.Registries, reg.Name)
	}

	return host
}

func MapDockerContainer(environment string, hostId string, hostName string, c types.Container, cj types.ContainerJSON) models.DockerContainer {
	container := models.DockerContainer{
		Id:           c.ID,
		HostId:       models.Hash(hostName),
		HostName:     hostName,
		Image:        c.Image,
		Command:      c.Command,
		State:        c.State,
		Status:       c.Status,
		Path:         cj.ContainerJSONBase.Path,
		Args:         cj.ContainerJSONBase.Args,
		Name:         cj.ContainerJSONBase.Name,
		RestartCount: cj.ContainerJSONBase.RestartCount,
		PortBindings: make(map[string]string),
		Collected:    time.Now().UTC(),
		Environment:  environment,
	}

	container.Labels = make(map[string]string)
	for key, value := range c.Labels {
		k := strings.Replace(key, ".", "_", -1)
		container.Labels[k] = value
	}

	container.Created, _ = time.Parse(time.RFC3339, cj.ContainerJSONBase.Created)
	if len(container.Name) > 1 {
		container.Name = container.Name[1:len(container.Name)]
	}

	if cj.Config != nil {
		container.Env = cj.Config.Env
	}

	if cj.ContainerJSONBase.State != nil {

		container.StartedAt, _ = time.Parse(time.RFC3339, cj.ContainerJSONBase.State.StartedAt)
		container.FinishedAt, _ = time.Parse(time.RFC3339, cj.ContainerJSONBase.State.FinishedAt)
		container.ExitCode = cj.ContainerJSONBase.State.ExitCode
		container.Error = cj.ContainerJSONBase.State.Error
	}

	if cj.ContainerJSONBase.HostConfig != nil {
		container.NetworkMode = string(cj.ContainerJSONBase.HostConfig.NetworkMode)
		container.RestartPolicy = cj.ContainerJSONBase.HostConfig.RestartPolicy.Name
		for key, val := range cj.ContainerJSONBase.HostConfig.PortBindings {
			if len(val) > 0 {
				container.PortBindings[string(key)] = val[0].HostPort
			}
		}
	}

	// use first Host port bind as container Port
	// if multiple ports are present search for gliderlabs/registrator meta
	container.Port = GetPortFromEnv(container.PortBindings, container.Env)

	return container
}

func GetPortFromEnv(portBindings map[string]string, env []string) string {
	port := ""
	if len(portBindings) == 0 {
		return port
	} else if len(portBindings) == 1 {
		for _, val := range portBindings {
			if len(val) > 0 {
				return val
			}
		}
	}
	if len(env) == 0 {
		for _, val := range portBindings {
			if len(val) > 0 {
				return val
			}
		}
	}
	for _, val := range portBindings {
		if len(val) > 0 {
			registratorMeta := "SERVICE_" + val + "_NAME"
			for _, e := range env {
				if strings.Contains(e, registratorMeta) {
					return val
				}
			}
		}
	}
	if port == "" {
		for _, val := range portBindings {
			if len(val) > 0 {
				return val
			}
		}
	}

	return port
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"crypto/sha1"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/bitrise-io/go-utils/cmdex"
	"github.com/bitrise-io/go-utils/pathutil"
)

var (
	gIsDebugMode = false
)

// StepParamsModel ...
type StepParamsModel struct {
	Paths                []string
	CacheUploadURL       string
	CompareCacheInfoPath string
	IsDebugMode          bool
}

// CacheContentModel ...
type CacheContentModel struct {
	DestinationPath       string `json:"destination_path"`
	RelativePathInArchive string `json:"relative_path_in_archive"`
}

// CacheInfosModel ...
type CacheInfosModel struct {
	Fingerprint string              `json:"fingerprint"`
	Contents    []CacheContentModel `json:"cache_contents"`
}

func readCacheInfoFromFile(filePth string) (CacheInfosModel, error) {
	jsonBytes, err := ioutil.ReadFile(filePth)
	if err != nil {
		return CacheInfosModel{}, fmt.Errorf("Failed to read file: %s", err)
	}
	var cacheInfo CacheInfosModel
	if err := json.Unmarshal(jsonBytes, &cacheInfo); err != nil {
		return CacheInfosModel{}, fmt.Errorf("Failed to parse JSON: %s", err)
	}
	return cacheInfo, nil
}

// CreateStepParamsFromEnvs ...
func CreateStepParamsFromEnvs() (StepParamsModel, error) {
	cacheDirs := os.Getenv("cache_paths")
	cacheUploadURL := os.Getenv("cache_upload_url")

	if cacheDirs == "" {
		return StepParamsModel{}, errors.New("No cache_paths input specified")
	}
	if cacheUploadURL == "" {
		return StepParamsModel{}, errors.New("No cache_upload_url input specified")
	}

	stepParams := StepParamsModel{
		CacheUploadURL:       cacheUploadURL,
		CompareCacheInfoPath: os.Getenv("compare_cache_info_path"),
		IsDebugMode:          os.Getenv("is_debug_mode") == "true",
	}

	scanner := bufio.NewScanner(strings.NewReader(cacheDirs))
	for scanner.Scan() {
		aCachePth := scanner.Text()
		if aCachePth != "" {
			stepParams.Paths = append(stepParams.Paths, aCachePth)
		}
	}
	if err := scanner.Err(); err != nil {
		return StepParamsModel{}, fmt.Errorf("Failed to scan the the cache_paths input: %s", err)
	}

	return stepParams, nil
}

func fingerprintSourceStringOfFile(pth string, fileInfo os.FileInfo) string {
	return fmt.Sprintf("[%s]-[%d]-[%dB]-[0x%o]", pth, fileInfo.ModTime().Unix(), fileInfo.Size(), fileInfo.Mode())
}

// fingerprintOfPaths ...
func fingerprintOfPaths(pths []string) ([]byte, error) {
	fingerprintHash := sha1.New()
	for _, aPath := range pths {
		if aPath == "" {
			continue
		}
		aPath = path.Clean(aPath)
		if aPath == "/" {
			return []byte{}, errors.New("Failed to check the specified path: caching the whole root (/) is forbidden (path was '/')")
		}
		fileInfo, isExist, err := pathutil.PathCheckAndInfos(aPath)
		if err != nil {
			return []byte{}, fmt.Errorf("Failed to check the specified path: %s", err)
		}
		if !isExist {
			return []byte{}, errors.New("Specified path does not exist")
		}

		if fileInfo.IsDir() {
			err := filepath.Walk(aPath, func(aPath string, aFileInfo os.FileInfo, walkErr error) error {
				if walkErr != nil {
					log.Printf(" (!) Error checking file (%s): %s", aPath, walkErr)
				}
				if aFileInfo.IsDir() {
					// directory - skipping
					return nil
				}
				if !aFileInfo.Mode().IsRegular() {
					if gIsDebugMode {
						log.Printf(" (i) File (%s) is not a regular file (it's a symlink, a device, or something similar) - skipping", aPath)
					}
					return nil
				}
				fileFingerprintSource := fingerprintSourceStringOfFile(aPath, aFileInfo)
				if gIsDebugMode {
					log.Printf(" * fileFingerprintSource (%s): %#v", aPath, fileFingerprintSource)
				}
				if _, err := io.WriteString(fingerprintHash, fileFingerprintSource); err != nil {
					return fmt.Errorf("Failed to write fingerprint source string (%s) to fingerprint hash: %s",
						fileFingerprintSource, err)
				}
				return nil
			})
			if err != nil {
				return []byte{}, fmt.Errorf("Failed to walk through the specified directory (%s): %s", aPath, err)
			}
		} else {
			fileFingerprintSource := fingerprintSourceStringOfFile(aPath, fileInfo)
			if gIsDebugMode {
				log.Printf(" -> fileFingerprintSource (%s): %#v", aPath, fileFingerprintSource)
			}
			if _, err := io.WriteString(fingerprintHash, fileFingerprintSource); err != nil {
				return []byte{}, fmt.Errorf("Failed to write fingerprint source string (%s) to fingerprint hash: %s",
					fileFingerprintSource, err)
			}
		}
	}

	return fingerprintHash.Sum(nil), nil
}

func cleanupCachePaths(requestedCachePaths []string) []string {
	filteredPaths := []string{}
	for _, aOrigPth := range requestedCachePaths {
		if aOrigPth == "" {
			continue
		}
		aPath := path.Clean(aOrigPth)
		if aPath == "/" {
			log.Println(" (!) Skipping: Failed to check the specified path: path was '/' - caching the whole root (/) directory is")
			continue
		}
		fileInfo, isExist, err := pathutil.PathCheckAndInfos(aPath)
		if err != nil {
			log.Printf(" (!) Skipping (%s): Failed to check the specified path: %s", aOrigPth, err)
			continue
		}
		if !isExist {
			log.Printf(" (!) Skipping (%s): Specified path does not exist", aOrigPth)
			continue
		}
		if !fileInfo.IsDir() {
			if !fileInfo.Mode().IsRegular() {
				log.Printf(" (i) File (%s) is not a regular file (it's a symlink, a device, or something similar) - skipping", aOrigPth)
				continue
			}
		}

		filteredPaths = append(filteredPaths, aPath)
	}
	return filteredPaths
}

func createCacheArchiveFromPaths(pathsToCache []string, archiveContentFingerprint string) (string, error) {
	cacheArchiveTmpBaseDirPth, err := pathutil.NormalizedOSTempDirPath("")
	if err != nil {
		return "", fmt.Errorf("Failed to create temporary Cache Archive directory: %s", err)
	}
	if gIsDebugMode {
		fmt.Println()
		log.Printf("=> cacheArchiveTmpBaseDirPth: %s", cacheArchiveTmpBaseDirPth)
		fmt.Println()
	}

	cacheContentDirName := "content"
	cacheContentDirPath := filepath.Join(cacheArchiveTmpBaseDirPth, cacheContentDirName)
	if err := pathutil.EnsureDirExist(cacheContentDirPath); err != nil {
		return "", fmt.Errorf("Failed to create Cache Content directory: %s", err)
	}

	cacheInfo := CacheInfosModel{
		Fingerprint: archiveContentFingerprint,
		Contents:    []CacheContentModel{},
	}
	for idx, aPath := range pathsToCache {
		aPath = path.Clean(aPath)
		fileInfo, isExist, err := pathutil.PathCheckAndInfos(aPath)
		if err != nil {
			return "", fmt.Errorf("Failed to check path (%s): %s", aPath, err)
		}
		if !isExist {
			return "", fmt.Errorf("Path does not exist: %s", aPath)
		}

		archiveCopyRsyncParams := []string{}
		itemRelPathInArchive := fmt.Sprintf("c-%d", idx)

		if fileInfo.IsDir() {
			archiveCopyRsyncParams = []string{"-avhP", aPath + "/", filepath.Join(cacheContentDirPath, itemRelPathInArchive+"/")}
		} else {
			archiveCopyRsyncParams = []string{"-avhP", aPath, filepath.Join(cacheContentDirPath, itemRelPathInArchive)}
		}

		cacheInfo.Contents = append(cacheInfo.Contents, CacheContentModel{
			DestinationPath:       aPath,
			RelativePathInArchive: itemRelPathInArchive,
		})

		if gIsDebugMode {
			log.Printf(" $ rsync %s", archiveCopyRsyncParams)
		}
		if fullOut, err := cmdex.RunCommandAndReturnCombinedStdoutAndStderr("rsync", archiveCopyRsyncParams...); err != nil {
			log.Printf(" [!] Failed to sync archive target (%s), full output (stdout & stderr) was: %s", aPath, fullOut)
			return "", fmt.Errorf("Failed to sync archive target (%s): %s", aPath, err)
		}
	}

	// store CacheInfo into the cache content dir
	jsonBytes, err := json.Marshal(cacheInfo)
	if err != nil {
		return "", fmt.Errorf("Failed to generate Cache Info JSON: %s", err)
	}
	cacheInfoFilePath := filepath.Join(cacheContentDirPath, "cache-info.json")
	if err := ioutil.WriteFile(cacheInfoFilePath, jsonBytes, 0666); err != nil {
		return "", fmt.Errorf("Failed to write Cache Info JSON to file (%s): %s", cacheInfoFilePath, err)
	}

	cacheArchiveFileName := "cache.tar.gz"
	cacheArchiveFilePath := filepath.Join(cacheArchiveTmpBaseDirPth, cacheArchiveFileName)
	tarCmdParams := []string{"-cvzf", cacheArchiveFilePath, "."}
	if gIsDebugMode {
		log.Printf(" $ tar %s", tarCmdParams)
	}
	if fullOut, err := cmdex.RunCommandInDirAndReturnCombinedStdoutAndStderr(cacheContentDirPath, "tar", tarCmdParams...); err != nil {
		log.Printf(" [!] Failed to create cache archive, full output (stdout & stderr) was: %s", fullOut)
		return "", fmt.Errorf("Failed to create cache archive, error was: %s", err)
	}

	return cacheArchiveFilePath, nil
}

func _tryToUploadArchive(stepParams StepParamsModel, archiveFilePath string) error {
	archFile, err := os.Open(archiveFilePath)
	if err != nil {
		return fmt.Errorf("Failed to open archive file for upload (%s): %s", archiveFilePath, err)
	}
	isFileCloseRequired := true
	defer func() {
		if !isFileCloseRequired {
			return
		}
		if err := archFile.Close(); err != nil {
			log.Printf(" (!) Failed to close archive file (%s): %s", archiveFilePath, err)
		}
	}()

	fileInfo, err := archFile.Stat()
	if err != nil {
		return fmt.Errorf("Failed to get File Stats of the Archive file (%s): %s", archiveFilePath, err)
	}
	fileSize := fileInfo.Size()

	req, err := http.NewRequest("PUT", stepParams.CacheUploadURL, archFile)
	if err != nil {
		return fmt.Errorf("Failed to create upload request: %s", err)
	}

	// req.Header.Set("Content-Type", "application/octet-stream")
	// req.Header.Add("Content-Length", strconv.FormatInt(fileSize, 10))
	req.ContentLength = fileSize
	if gIsDebugMode {
		log.Printf("=> req: %#v", req)
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return fmt.Errorf("Failed to upload: %s", err)
	}
	isFileCloseRequired = false
	defer func() {
		if err := resp.Body.Close(); err != nil {
			log.Printf(" [!] Failed to close response body: %s", err)
		}
	}()

	responseBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return fmt.Errorf("Failed to read response: %s", err)
	}
	log.Printf("=> Upload response: %s", responseBytes)

	if resp.StatusCode != 200 {
		return fmt.Errorf("Failed to upload file, response code was: %d", resp.StatusCode)
	}
	log.Printf("=> Upload response code: %d", resp.StatusCode)

	return nil
}

func uploadArchive(stepParams StepParamsModel, archiveFilePath string) error {
	if err := _tryToUploadArchive(stepParams, archiveFilePath); err != nil {
		fmt.Println()
		log.Printf(" ===> (!) First upload attempt failed, retrying...")
		fmt.Println()
		time.Sleep(3000 * time.Millisecond)
		return _tryToUploadArchive(stepParams, archiveFilePath)
	}
	return nil
}

func main() {
	stepParams, err := CreateStepParamsFromEnvs()
	if err != nil {
		log.Fatalf(" [!] Input error : %s", err)
	}
	gIsDebugMode = stepParams.IsDebugMode

	if gIsDebugMode {
		log.Printf("=> stepParams: %#v", stepParams)
	}

	log.Printf("=> Oritinal list of paths to cache: %s", stepParams.Paths)
	stepParams.Paths = cleanupCachePaths(stepParams.Paths)
	log.Printf("=> Filtered paths to cache: %s", stepParams.Paths)

	if len(stepParams.Paths) < 1 {
		log.Println("No paths specified to be cached, stopping.")
		os.Exit(3)
	}

	// fingerprint
	pthsFingerprint, err := fingerprintOfPaths(stepParams.Paths)
	if err != nil {
		log.Fatalf(" [!] Failed to calculate fingerprint: %s", err)
	}
	if len(pthsFingerprint) < 1 {
		log.Fatal(" [!] Failed to calculate fingerprint: empty fingerprint generated")
	}
	fingerprintBase16Str := fmt.Sprintf("%x", pthsFingerprint)
	log.Printf("=> Calculated Fingerprint (base 16): %s", fingerprintBase16Str)

	// compare fingerprints
	if stepParams.CompareCacheInfoPath != "" {
		if gIsDebugMode {
			log.Printf("Comparing fingerprint with cache info from: %s", stepParams.CompareCacheInfoPath)
		}
		cacheInfo, err := readCacheInfoFromFile(stepParams.CompareCacheInfoPath)
		if err != nil {
			log.Printf(" [!] Failed to read Cache Info for compare: %s", err)
		} else {
			if cacheInfo.Fingerprint == fingerprintBase16Str {
				log.Println(" => (i) Fingerprint matches the original one, no need to update Cache - DONE")
				return
			}
			log.Printf(" (i) Fingerprint (%s) does not match the original one (%s), Cache update required", fingerprintBase16Str, cacheInfo.Fingerprint)
		}
	} else {
		log.Println("No base Cache Info found for compare - New cache will be created")
	}

	archiveFilePath, err := createCacheArchiveFromPaths(stepParams.Paths, fingerprintBase16Str)
	if err != nil {
		log.Fatalf(" [!] Failed to create Cache Archive: %s", err)
	}
	log.Printf(" => archiveFilePath: %s", archiveFilePath)

	if err := uploadArchive(stepParams, archiveFilePath); err != nil {
		log.Fatalf(" [!] Failed to upload Cache Archive: %s", err)
	}

	log.Println(" => DONE")
}
<commit_msg>Log: Uploading ...<commit_after>package main

import (
	"bufio"
	"crypto/sha1"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/bitrise-io/go-utils/cmdex"
	"github.com/bitrise-io/go-utils/pathutil"
)

var (
	gIsDebugMode = false
)

// StepParamsModel ...
type StepParamsModel struct {
	Paths                []string
	CacheUploadURL       string
	CompareCacheInfoPath string
	IsDebugMode          bool
}

// CacheContentModel ...
type CacheContentModel struct {
	DestinationPath       string `json:"destination_path"`
	RelativePathInArchive string `json:"relative_path_in_archive"`
}

// CacheInfosModel ...
type CacheInfosModel struct {
	Fingerprint string              `json:"fingerprint"`
	Contents    []CacheContentModel `json:"cache_contents"`
}

func readCacheInfoFromFile(filePth string) (CacheInfosModel, error) {
	jsonBytes, err := ioutil.ReadFile(filePth)
	if err != nil {
		return CacheInfosModel{}, fmt.Errorf("Failed to read file: %s", err)
	}
	var cacheInfo CacheInfosModel
	if err := json.Unmarshal(jsonBytes, &cacheInfo); err != nil {
		return CacheInfosModel{}, fmt.Errorf("Failed to parse JSON: %s", err)
	}
	return cacheInfo, nil
}

// CreateStepParamsFromEnvs ...
func CreateStepParamsFromEnvs() (StepParamsModel, error) {
	cacheDirs := os.Getenv("cache_paths")
	cacheUploadURL := os.Getenv("cache_upload_url")

	if cacheDirs == "" {
		return StepParamsModel{}, errors.New("No cache_paths input specified")
	}
	if cacheUploadURL == "" {
		return StepParamsModel{}, errors.New("No cache_upload_url input specified")
	}

	stepParams := StepParamsModel{
		CacheUploadURL:       cacheUploadURL,
		CompareCacheInfoPath: os.Getenv("compare_cache_info_path"),
		IsDebugMode:          os.Getenv("is_debug_mode") == "true",
	}

	scanner := bufio.NewScanner(strings.NewReader(cacheDirs))
	for scanner.Scan() {
		aCachePth := scanner.Text()
		if aCachePth != "" {
			stepParams.Paths = append(stepParams.Paths, aCachePth)
		}
	}
	if err := scanner.Err(); err != nil {
		return StepParamsModel{}, fmt.Errorf("Failed to scan the the cache_paths input: %s", err)
	}

	return stepParams, nil
}

func fingerprintSourceStringOfFile(pth string, fileInfo os.FileInfo) string {
	return fmt.Sprintf("[%s]-[%d]-[%dB]-[0x%o]", pth, fileInfo.ModTime().Unix(), fileInfo.Size(), fileInfo.Mode())
}

// fingerprintOfPaths ...
func fingerprintOfPaths(pths []string) ([]byte, error) {
	fingerprintHash := sha1.New()
	for _, aPath := range pths {
		if aPath == "" {
			continue
		}
		aPath = path.Clean(aPath)
		if aPath == "/" {
			return []byte{}, errors.New("Failed to check the specified path: caching the whole root (/) is forbidden (path was '/')")
		}
		fileInfo, isExist, err := pathutil.PathCheckAndInfos(aPath)
		if err != nil {
			return []byte{}, fmt.Errorf("Failed to check the specified path: %s", err)
		}
		if !isExist {
			return []byte{}, errors.New("Specified path does not exist")
		}

		if fileInfo.IsDir() {
			err := filepath.Walk(aPath, func(aPath string, aFileInfo os.FileInfo, walkErr error) error {
				if walkErr != nil {
					log.Printf(" (!) Error checking file (%s): %s", aPath, walkErr)
				}
				if aFileInfo.IsDir() {
					// directory - skipping
					return nil
				}
				if !aFileInfo.Mode().IsRegular() {
					if gIsDebugMode {
						log.Printf(" (i) File (%s) is not a regular file (it's a symlink, a device, or something similar) - skipping", aPath)
					}
					return nil
				}
				fileFingerprintSource := fingerprintSourceStringOfFile(aPath, aFileInfo)
				if gIsDebugMode {
					log.Printf(" * fileFingerprintSource (%s): %#v", aPath, fileFingerprintSource)
				}
				if _, err := io.WriteString(fingerprintHash, fileFingerprintSource); err != nil {
					return fmt.Errorf("Failed to write fingerprint source string (%s) to fingerprint hash: %s",
						fileFingerprintSource, err)
				}
				return nil
			})
			if err != nil {
				return []byte{}, fmt.Errorf("Failed to walk through the specified directory (%s): %s", aPath, err)
			}
		} else {
			fileFingerprintSource := fingerprintSourceStringOfFile(aPath, fileInfo)
			if gIsDebugMode {
				log.Printf(" -> fileFingerprintSource (%s): %#v", aPath, fileFingerprintSource)
			}
			if _, err := io.WriteString(fingerprintHash, fileFingerprintSource); err != nil {
				return []byte{}, fmt.Errorf("Failed to write fingerprint source string (%s) to fingerprint hash: %s",
					fileFingerprintSource, err)
			}
		}
	}

	return fingerprintHash.Sum(nil), nil
}

func cleanupCachePaths(requestedCachePaths []string) []string {
	filteredPaths := []string{}
	for _, aOrigPth := range requestedCachePaths {
		if aOrigPth == "" {
			continue
		}
		aPath := path.Clean(aOrigPth)
		if aPath == "/" {
			log.Println(" (!) Skipping: Failed to check the specified path: path was '/' - caching the whole root (/) directory is")
			continue
		}
		fileInfo, isExist, err := pathutil.PathCheckAndInfos(aPath)
		if err != nil {
			log.Printf(" (!) Skipping (%s): Failed to check the specified path: %s", aOrigPth, err)
			continue
		}
		if !isExist {
			log.Printf(" (!) Skipping (%s): Specified path does not exist", aOrigPth)
			continue
		}
		if !fileInfo.IsDir() {
			if !fileInfo.Mode().IsRegular() {
				log.Printf(" (i) File (%s) is not a regular file (it's a symlink, a device, or something similar) - skipping", aOrigPth)
				continue
			}
		}

		filteredPaths = append(filteredPaths, aPath)
	}
	return filteredPaths
}

func createCacheArchiveFromPaths(pathsToCache []string, archiveContentFingerprint string) (string, error) {
	cacheArchiveTmpBaseDirPth, err := pathutil.NormalizedOSTempDirPath("")
	if err != nil {
		return "", fmt.Errorf("Failed to create temporary Cache Archive directory: %s", err)
	}
	if gIsDebugMode {
		fmt.Println()
		log.Printf("=> cacheArchiveTmpBaseDirPth: %s", cacheArchiveTmpBaseDirPth)
		fmt.Println()
	}

	cacheContentDirName := "content"
	cacheContentDirPath := filepath.Join(cacheArchiveTmpBaseDirPth, cacheContentDirName)
	if err := pathutil.EnsureDirExist(cacheContentDirPath); err != nil {
		return "", fmt.Errorf("Failed to create Cache Content directory: %s", err)
	}

	cacheInfo := CacheInfosModel{
		Fingerprint: archiveContentFingerprint,
		Contents:    []CacheContentModel{},
	}
	for idx, aPath := range pathsToCache {
		aPath = path.Clean(aPath)
		fileInfo, isExist, err := pathutil.PathCheckAndInfos(aPath)
		if err != nil {
			return "", fmt.Errorf("Failed to check path (%s): %s", aPath, err)
		}
		if !isExist {
			return "", fmt.Errorf("Path does not exist: %s", aPath)
		}

		archiveCopyRsyncParams := []string{}
		itemRelPathInArchive := fmt.Sprintf("c-%d", idx)

		if fileInfo.IsDir() {
			archiveCopyRsyncParams = []string{"-avhP", aPath + "/", filepath.Join(cacheContentDirPath, itemRelPathInArchive+"/")}
		} else {
			archiveCopyRsyncParams = []string{"-avhP", aPath, filepath.Join(cacheContentDirPath, itemRelPathInArchive)}
		}

		cacheInfo.Contents = append(cacheInfo.Contents, CacheContentModel{
			DestinationPath:       aPath,
			RelativePathInArchive: itemRelPathInArchive,
		})

		if gIsDebugMode {
			log.Printf(" $ rsync %s", archiveCopyRsyncParams)
		}
		if fullOut, err := cmdex.RunCommandAndReturnCombinedStdoutAndStderr("rsync", archiveCopyRsyncParams...); err != nil {
			log.Printf(" [!] Failed to sync archive target (%s), full output (stdout & stderr) was: %s", aPath, fullOut)
			return "", fmt.Errorf("Failed to sync archive target (%s): %s", aPath, err)
		}
	}

	// store CacheInfo into the cache content dir
	jsonBytes, err := json.Marshal(cacheInfo)
	if err != nil {
		return "", fmt.Errorf("Failed to generate Cache Info JSON: %s", err)
	}
	cacheInfoFilePath := filepath.Join(cacheContentDirPath, "cache-info.json")
	if err := ioutil.WriteFile(cacheInfoFilePath, jsonBytes, 0666); err != nil {
		return "", fmt.Errorf("Failed to write Cache Info JSON to file (%s): %s", cacheInfoFilePath, err)
	}

	cacheArchiveFileName := "cache.tar.gz"
	cacheArchiveFilePath := filepath.Join(cacheArchiveTmpBaseDirPth, cacheArchiveFileName)
	tarCmdParams := []string{"-cvzf", cacheArchiveFilePath, "."}
	if gIsDebugMode {
		log.Printf(" $ tar %s", tarCmdParams)
	}
	if fullOut, err := cmdex.RunCommandInDirAndReturnCombinedStdoutAndStderr(cacheContentDirPath, "tar", tarCmdParams...); err != nil {
		log.Printf(" [!] Failed to create cache archive, full output (stdout & stderr) was: %s", fullOut)
		return "", fmt.Errorf("Failed to create cache archive, error was: %s", err)
	}

	return cacheArchiveFilePath, nil
}

func _tryToUploadArchive(stepParams StepParamsModel, archiveFilePath string) error {
	archFile, err := os.Open(archiveFilePath)
	if err != nil {
		return fmt.Errorf("Failed to open archive file for upload (%s): %s", archiveFilePath, err)
	}
	isFileCloseRequired := true
	defer func() {
		if !isFileCloseRequired {
			return
		}
		if err := archFile.Close(); err != nil {
			log.Printf(" (!) Failed to close archive file (%s): %s", archiveFilePath, err)
		}
	}()

	fileInfo, err := archFile.Stat()
	if err != nil {
		return fmt.Errorf("Failed to get File Stats of the Archive file (%s): %s", archiveFilePath, err)
	}
	fileSize := fileInfo.Size()

	req, err := http.NewRequest("PUT", stepParams.CacheUploadURL, archFile)
	if err != nil {
		return fmt.Errorf("Failed to create upload request: %s", err)
	}

	// req.Header.Set("Content-Type", "application/octet-stream")
	// req.Header.Add("Content-Length", strconv.FormatInt(fileSize, 10))
	req.ContentLength = fileSize
	if gIsDebugMode {
		log.Printf("=> req: %#v", req)
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return fmt.Errorf("Failed to upload: %s", err)
	}
	isFileCloseRequired = false
	defer func() {
		if err := resp.Body.Close(); err != nil {
			log.Printf(" [!] Failed to close response body: %s", err)
		}
	}()

	responseBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return fmt.Errorf("Failed to read response: %s", err)
	}
	log.Printf("=> Upload response: %s", responseBytes)

	if resp.StatusCode != 200 {
		return fmt.Errorf("Failed to upload file, response code was: %d", resp.StatusCode)
	}
	log.Printf("=> Upload response code: %d", resp.StatusCode)

	return nil
}

func uploadArchive(stepParams StepParamsModel, archiveFilePath string) error {
	log.Println("=> Uploading ...")
	if err := _tryToUploadArchive(stepParams, archiveFilePath); err != nil {
		fmt.Println()
		log.Printf(" ===> (!) First upload attempt failed, retrying...")
		fmt.Println()
		time.Sleep(3000 * time.Millisecond)
		return _tryToUploadArchive(stepParams, archiveFilePath)
	}
	return nil
}

func main() {
	stepParams, err := CreateStepParamsFromEnvs()
	if err != nil {
		log.Fatalf(" [!] Input error : %s", err)
	}
	gIsDebugMode = stepParams.IsDebugMode

	if gIsDebugMode {
		log.Printf("=> stepParams: %#v", stepParams)
	}

	log.Printf("=> Oritinal list of paths to cache: %s", stepParams.Paths)
	stepParams.Paths = cleanupCachePaths(stepParams.Paths)
	log.Printf("=> Filtered paths to cache: %s", stepParams.Paths)

	if len(stepParams.Paths) < 1 {
		log.Println("No paths specified to be cached, stopping.")
		os.Exit(3)
	}

	// fingerprint
	pthsFingerprint, err := fingerprintOfPaths(stepParams.Paths)
	if err != nil {
		log.Fatalf(" [!] Failed to calculate fingerprint: %s", err)
	}
	if len(pthsFingerprint) < 1 {
		log.Fatal(" [!] Failed to calculate fingerprint: empty fingerprint generated")
	}
	fingerprintBase16Str := fmt.Sprintf("%x", pthsFingerprint)
	log.Printf("=> Calculated Fingerprint (base 16): %s", fingerprintBase16Str)

	// compare fingerprints
	if stepParams.CompareCacheInfoPath != "" {
		if gIsDebugMode {
			log.Printf("Comparing fingerprint with cache info from: %s", stepParams.CompareCacheInfoPath)
		}
		cacheInfo, err := readCacheInfoFromFile(stepParams.CompareCacheInfoPath)
		if err != nil {
			log.Printf(" [!] Failed to read Cache Info for compare: %s", err)
		} else {
			if cacheInfo.Fingerprint == fingerprintBase16Str {
				log.Println(" => (i) Fingerprint matches the original one, no need to update Cache - DONE")
				return
			}
			log.Printf(" (i) Fingerprint (%s) does not match the original one (%s), Cache update required", fingerprintBase16Str, cacheInfo.Fingerprint)
		}
	} else {
		log.Println("No base Cache Info found for compare - New cache will be created")
	}

	archiveFilePath, err := createCacheArchiveFromPaths(stepParams.Paths, fingerprintBase16Str)
	if err != nil {
		log.Fatalf(" [!] Failed to create Cache Archive: %s", err)
	}
	log.Printf(" => archiveFilePath: %s", archiveFilePath)

	if err := uploadArchive(stepParams, archiveFilePath); err != nil {
		log.Fatalf(" [!] Failed to upload Cache Archive: %s", err)
	}

	log.Println(" => DONE")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/gorilla/mux"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"pttbbs"
	"pttweb/article"
	"pttweb/cache"
	"runtime/pprof"
	"strconv"
	"strings"
	"text/template"
	"time"
)

var (
	ArticleCacheTimeout = time.Minute * 10
)

var ptt pttbbs.Pttbbs
var router *mux.Router
var tmpl TemplateMap
var cacheMgr *cache.CacheManager

var bindAddress string
var boarddAddress string
var memcachedAddress string
var templateDir string
var cpuProfile string
var memProfile string

func init() {
	flag.StringVar(&bindAddress, "bind", "127.0.0.1:8891", "bind address of the server (host:port)")
	flag.StringVar(&boarddAddress, "boardd", "", "boardd address (host:port)")
	flag.StringVar(&memcachedAddress, "memcached", "", "memcached address (host:port)")
	flag.StringVar(&templateDir, "tmpldir", "templates", "template directory, loads all *.html")
	flag.StringVar(&cpuProfile, "cpuprofile", "", "write cpu profile to file")
	flag.StringVar(&memProfile, "memprofile", "", "write memory profile to this file")
}

func main() {
	flag.Parse()

	// CPU Profiling
	if cpuProfile != "" {
		f, err := os.Create(cpuProfile)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	// Init RemotePtt
	if boarddAddress == "" {
		panic("boardd address not specified")
	}
	ptt = pttbbs.NewRemotePtt(boarddAddress)

	// Init cache manager
	if memcachedAddress == "" {
		panic("memcached address not specified")
	}
	cacheMgr = cache.NewCacheManager(memcachedAddress)

	// Load templates
	if t, err := loadTemplates(templateDir, templateFiles); err != nil {
		panic(err)
	} else {
		tmpl = t
	}

	// Init router
	router = createRouter()
	http.Handle("/", router)

	go func() {
		if err := http.ListenAndServe(bindAddress, nil); err != nil {
			log.Fatal("ListenAndServe: ", err)
			os.Exit(1)
		}
	}()

	progExit := make(chan os.Signal)
	signal.Notify(progExit, os.Interrupt)
	<-progExit

	if memProfile != "" {
		f, err := os.Create(memProfile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.WriteHeapProfile(f)
		f.Close()
	}
}

func createRouter() *mux.Router {
	router := mux.NewRouter()
	router.HandleFunc(`/cls/{bid:[0-9]+}`, errorWrapperHandler(handleCls)).Name("classlist")
	router.HandleFunc(`/bbs/{brdname:[A-Za-z][0-9a-zA-Z_\.\-]+}{x:/?}`, errorWrapperHandler(handleBbsIndexRedirect))
	router.HandleFunc(`/bbs/{brdname:[A-Za-z][0-9a-zA-Z_\.\-]+}/index.html`, errorWrapperHandler(handleBbs)).Name("bbsindex")
	router.HandleFunc(`/bbs/{brdname:[A-Za-z][0-9a-zA-Z_\.\-]+}/index{page:\d+}.html`, errorWrapperHandler(handleBbs)).Name("bbsindex_page")
	router.HandleFunc(`/bbs/{brdname:[A-Za-z][0-9a-zA-Z_\.\-]+}/{filename:[MG]\.\d+(\.A\.[0-9A-F]+)?}.html`, errorWrapperHandler(handleArticle)).Name("bbsarticle")
	return router
}

func templateFuncMap() template.FuncMap {
	return template.FuncMap{
		"route_bbsindex": func(b pttbbs.Board) (*url.URL, error) {
			return router.Get("bbsindex").URLPath("brdname", b.BrdName)
		},
		"route_classlist": func(b pttbbs.Board) (*url.URL, error) {
			return router.Get("classlist").URLPath("bid", strconv.Itoa(b.Bid))
		},
		"valid_article": func(a pttbbs.Article) bool {
			return !strings.HasPrefix(a.FileName, ".")
		},
		"route_bbsarticle": func(brdname, filename string) (*url.URL, error) {
			return router.Get("bbsarticle").URLPath("brdname", brdname, "filename", filename)
		},
		"route": func(where string, attrs ...string) (*url.URL, error) {
			return router.Get(where).URLPath(attrs...)
		},
		"colored_counter": colored_counter,
		"post_mark":       post_mark,
	}
}

func setCommonResponseHeaders(w http.ResponseWriter) {
	h := w.Header()
	h.Set("Server", "Cryophoenix")
	h.Set("Content-Type", "text/html; charset=utf-8")
}

func errorWrapperHandler(f func(http.ResponseWriter, *http.Request) error) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		setCommonResponseHeaders(w)
		if err := f(w, r); err != nil {
			log.Println(err)
			w.WriteHeader(http.StatusInternalServerError)
		}
	}
}

func handleNotFound(w http.ResponseWriter, r *http.Request) error {
	w.WriteHeader(http.StatusNotFound)
	return tmpl["notfound.html"].Execute(w, nil)
}

func handleCls(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)
	bid, err := strconv.Atoi(vars["bid"])
	if err != nil {
		return err
	}

	children, err := ptt.GetBoardChildren(bid)
	if err != nil {
		return err
	}

	boards := make([]pttbbs.Board, 0, 16)
	for _, bid := range children {
		if brd, err := ptt.GetBoard(bid); err == nil {
			if pttbbs.IsValidBrdName(brd.BrdName) && !brd.Over18 && !brd.Hidden {
				boards = append(boards, brd)
			}
		}
	}

	return tmpl["classlist.html"].Execute(w, map[string]interface{}{
		"Boards": boards,
	})
}

func handleBbsIndexRedirect(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)
	if url, err := router.Get("bbsindex").URLPath("brdname", vars["brdname"]); err != nil {
		return err
	} else {
		w.Header().Set("Location", url.String())
	}
	w.WriteHeader(http.StatusFound)
	return nil
}

func handleBbs(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)
	brdname := vars["brdname"]
	page := 0

	if pg, err := strconv.Atoi(vars["page"]); err == nil {
		page = pg
	}

	var err error
	params := make(map[string]interface{})

	bid, err := ptt.BrdName2Bid(brdname)
	if err != nil {
		return err
	}

	brd, err := ptt.GetBoard(bid)
	if err != nil {
		return err
	}
	params["Board"] = brd

	err = hasPermViewBoard(brd)
	if err != nil {
		return err
	}

	count, err := ptt.GetArticleCount(bid)
	if err != nil {
		return err
	}

	// Handle paging
	paging := NewPaging(20, count)
	if page == 0 {
		page = paging.LastPageNo()
		paging.SetPageNo(page)
	} else if err = paging.SetPageNo(page); err != nil {
		return err
	}

	// Fetch article list
	params["Articles"], err = ptt.GetArticleList(bid, paging.Cursor())
	if err != nil {
		return err
	}

	// Page links
	pageLink := func(i int) *url.URL {
		if i < 1 || i > paging.LastPageNo() {
			return nil
		}
		if u, err := router.Get("bbsindex_page").URLPath("brdname", brdname, "page", strconv.Itoa(i)); err == nil {
			return u
		}
		return nil
	}
	params["PrevPage"] = pageLink(page - 1)
	params["NextPage"] = pageLink(page + 1)
	params["FirstPage"] = pageLink(1)
	params["LastPage"], err = router.Get("bbsindex").URLPath("brdname", brdname)
	if err != nil {
		return err
	}

	return tmpl["bbsindex.html"].Execute(w, params)
}

func handleArticle(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)
	brdname := vars["brdname"]
	filename := vars["filename"]

	var err error

	bid, err := ptt.BrdName2Bid(brdname)
	if err != nil {
		return err
	}

	brd, err := ptt.GetBoard(bid)
	if err != nil {
		return err
	}

	err = hasPermViewBoard(brd)
	if err != nil {
		return err
	}

	// Render content
	resultChan := cacheMgr.Get(
		fmt.Sprintf("pttweb:bbs/%s/%s", brd.BrdName, filename),
		func(key string) (res cache.Result) {
			a := generateArticle(bid, filename)
			res.Expire = ArticleCacheTimeout
			res.Output, res.Err = a.EncodeToBytes()
			if res.Err != nil {
				res.Output = nil
			}
			return
		})

	result := <-resultChan
	var ar cache.Article
	if err = cache.GobDecode(result.Output, &ar); err != nil {
		return err
	}

	if !ar.IsValid {
		return handleNotFound(w, r)
	}

	return tmpl["bbsarticle.html"].Execute(w, map[string]interface{}{
		"Title":       ar.ParsedTitle,
		"Description": ar.PreviewContent,
		"Board":       brd,
		"ContentHtml": string(ar.ContentHtml),
	})
}

func generateArticle(bid int, filename string) (a cache.Article) {
	content, err := ptt.GetArticleContent(bid, filename)
	if err != nil || content == nil {
		return
	}

	ar := article.NewRenderer()
	buf, err := ar.Render(content)
	if err == nil {
		a.ParsedTitle = ar.ParsedTitle()
		a.PreviewContent = ar.PreviewContent()
		a.ContentHtml = buf.Bytes()
		a.IsValid = true
	}
	return
}

func hasPermViewBoard(brd pttbbs.Board) error {
	if !pttbbs.IsValidBrdName(brd.BrdName) || brd.Over18 || brd.Hidden {
		return fmt.Errorf("No permission: %s", brd.BrdName)
	}
	return nil
}

func boardlist(ptt pttbbs.Pttbbs, indent string, root int, loop map[int]bool) {
	if loop[root] {
		//fmt.Println(indent, "loop skipped")
		return
	}
	loop[root] = true

	children, err := ptt.GetBoardChildren(root)
	if err != nil {
		fmt.Print(err)
		return
	}

	for _, bid := range children {
		if brd, err := ptt.GetBoard(bid); err == nil {
			if !brd.Hidden {
				//fmt.Println(indent, bid, brd)
				if !brd.IsBoard {
					boardlist(ptt, indent+"  ", bid, loop)
				} else {
					fmt.Println(brd.BrdName)
				}
			}
		} else {
			//fmt.Println(indent, bid, err)
		}
	}
}
<commit_msg>pttweb: remove profiling.<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/gorilla/mux"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"pttbbs"
	"pttweb/article"
	"pttweb/cache"
	"runtime/pprof"
	"strconv"
	"strings"
	"text/template"
	"time"
)

var (
	ArticleCacheTimeout = time.Minute * 10
)

var ptt pttbbs.Pttbbs
var router *mux.Router
var tmpl TemplateMap
var cacheMgr *cache.CacheManager

var bindAddress string
var boarddAddress string
var memcachedAddress string
var templateDir string
var cpuProfile string
var memProfile string

func init() {
	flag.StringVar(&bindAddress, "bind", "127.0.0.1:8891", "bind address of the server (host:port)")
	flag.StringVar(&boarddAddress, "boardd", "", "boardd address (host:port)")
	flag.StringVar(&memcachedAddress, "memcached", "", "memcached address (host:port)")
	flag.StringVar(&templateDir, "tmpldir", "templates", "template directory, loads all *.html")
}

func main() {
	flag.Parse()

	// Init RemotePtt
	if boarddAddress == "" {
		panic("boardd address not specified")
	}
	ptt = pttbbs.NewRemotePtt(boarddAddress)

	// Init cache manager
	if memcachedAddress == "" {
		panic("memcached address not specified")
	}
	cacheMgr = cache.NewCacheManager(memcachedAddress)

	// Load templates
	if t, err := loadTemplates(templateDir, templateFiles); err != nil {
		panic(err)
	} else {
		tmpl = t
	}

	// Init router
	router = createRouter()
	http.Handle("/", router)

	go func() {
		if err := http.ListenAndServe(bindAddress, nil); err != nil {
			log.Fatal("ListenAndServe: ", err)
			os.Exit(1)
		}
	}()

	progExit := make(chan os.Signal)
	signal.Notify(progExit, os.Interrupt)
	<-progExit
}

func createRouter() *mux.Router {
	router := mux.NewRouter()
	router.HandleFunc(`/cls/{bid:[0-9]+}`, errorWrapperHandler(handleCls)).Name("classlist")
	router.HandleFunc(`/bbs/{brdname:[A-Za-z][0-9a-zA-Z_\.\-]+}{x:/?}`, errorWrapperHandler(handleBbsIndexRedirect))
	router.HandleFunc(`/bbs/{brdname:[A-Za-z][0-9a-zA-Z_\.\-]+}/index.html`, errorWrapperHandler(handleBbs)).Name("bbsindex")
	router.HandleFunc(`/bbs/{brdname:[A-Za-z][0-9a-zA-Z_\.\-]+}/index{page:\d+}.html`, errorWrapperHandler(handleBbs)).Name("bbsindex_page")
	router.HandleFunc(`/bbs/{brdname:[A-Za-z][0-9a-zA-Z_\.\-]+}/{filename:[MG]\.\d+(\.A\.[0-9A-F]+)?}.html`, errorWrapperHandler(handleArticle)).Name("bbsarticle")
	return router
}

func templateFuncMap() template.FuncMap {
	return template.FuncMap{
		"route_bbsindex": func(b pttbbs.Board) (*url.URL, error) {
			return router.Get("bbsindex").URLPath("brdname", b.BrdName)
		},
		"route_classlist": func(b pttbbs.Board) (*url.URL, error) {
			return router.Get("classlist").URLPath("bid", strconv.Itoa(b.Bid))
		},
		"valid_article": func(a pttbbs.Article) bool {
			return !strings.HasPrefix(a.FileName, ".")
		},
		"route_bbsarticle": func(brdname, filename string) (*url.URL, error) {
			return router.Get("bbsarticle").URLPath("brdname", brdname, "filename", filename)
		},
		"route": func(where string, attrs ...string) (*url.URL, error) {
			return router.Get(where).URLPath(attrs...)
		},
		"colored_counter": colored_counter,
		"post_mark":       post_mark,
	}
}

func setCommonResponseHeaders(w http.ResponseWriter) {
	h := w.Header()
	h.Set("Server", "Cryophoenix")
	h.Set("Content-Type", "text/html; charset=utf-8")
}

func errorWrapperHandler(f func(http.ResponseWriter, *http.Request) error) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		setCommonResponseHeaders(w)
		if err := f(w, r); err != nil {
			log.Println(err)
			w.WriteHeader(http.StatusInternalServerError)
		}
	}
}

func handleNotFound(w http.ResponseWriter, r *http.Request) error {
	w.WriteHeader(http.StatusNotFound)
	return tmpl["notfound.html"].Execute(w, nil)
}

func handleCls(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)
	bid, err := strconv.Atoi(vars["bid"])
	if err != nil {
		return err
	}

	children, err := ptt.GetBoardChildren(bid)
	if err != nil {
		return err
	}

	boards := make([]pttbbs.Board, 0, 16)
	for _, bid := range children {
		if brd, err := ptt.GetBoard(bid); err == nil {
			if pttbbs.IsValidBrdName(brd.BrdName) && !brd.Over18 && !brd.Hidden {
				boards = append(boards, brd)
			}
		}
	}

	return tmpl["classlist.html"].Execute(w, map[string]interface{}{
		"Boards": boards,
	})
}

func handleBbsIndexRedirect(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)
	if url, err := router.Get("bbsindex").URLPath("brdname", vars["brdname"]); err != nil {
		return err
	} else {
		w.Header().Set("Location", url.String())
	}
	w.WriteHeader(http.StatusFound)
	return nil
}

func handleBbs(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)
	brdname := vars["brdname"]
	page := 0

	if pg, err := strconv.Atoi(vars["page"]); err == nil {
		page = pg
	}

	var err error
	params := make(map[string]interface{})

	bid, err := ptt.BrdName2Bid(brdname)
	if err != nil {
		return err
	}

	brd, err := ptt.GetBoard(bid)
	if err != nil {
		return err
	}
	params["Board"] = brd

	err = hasPermViewBoard(brd)
	if err != nil {
		return err
	}

	count, err := ptt.GetArticleCount(bid)
	if err != nil {
		return err
	}

	// Handle paging
	paging := NewPaging(20, count)
	if page == 0 {
		page = paging.LastPageNo()
		paging.SetPageNo(page)
	} else if err = paging.SetPageNo(page); err != nil {
		return err
	}

	// Fetch article list
	params["Articles"], err = ptt.GetArticleList(bid, paging.Cursor())
	if err != nil {
		return err
	}

	// Page links
	pageLink := func(i int) *url.URL {
		if i < 1 || i > paging.LastPageNo() {
			return nil
		}
		if u, err := router.Get("bbsindex_page").URLPath("brdname", brdname, "page", strconv.Itoa(i)); err == nil {
			return u
		}
		return nil
	}
	params["PrevPage"] = pageLink(page - 1)
	params["NextPage"] = pageLink(page + 1)
	params["FirstPage"] = pageLink(1)
	params["LastPage"], err = router.Get("bbsindex").URLPath("brdname", brdname)
	if err != nil {
		return err
	}

	return tmpl["bbsindex.html"].Execute(w, params)
}

func handleArticle(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)
	brdname := vars["brdname"]
	filename := vars["filename"]

	var err error

	bid, err := ptt.BrdName2Bid(brdname)
	if err != nil {
		return err
	}

	brd, err := ptt.GetBoard(bid)
	if err != nil {
		return err
	}

	err = hasPermViewBoard(brd)
	if err != nil {
		return err
	}

	// Render content
	resultChan := cacheMgr.Get(
		fmt.Sprintf("pttweb:bbs/%s/%s", brd.BrdName, filename),
		func(key string) (res cache.Result) {
			a := generateArticle(bid, filename)
			res.Expire = ArticleCacheTimeout
			res.Output, res.Err = a.EncodeToBytes()
			if res.Err != nil {
				res.Output = nil
			}
			return
		})

	result := <-resultChan
	var ar cache.Article
	if err = cache.GobDecode(result.Output, &ar); err != nil {
		return err
	}

	if !ar.IsValid {
		return handleNotFound(w, r)
	}

	return tmpl["bbsarticle.html"].Execute(w, map[string]interface{}{
		"Title":       ar.ParsedTitle,
		"Description": ar.PreviewContent,
		"Board":       brd,
		"ContentHtml": string(ar.ContentHtml),
	})
}

func generateArticle(bid int, filename string) (a cache.Article) {
	content, err := ptt.GetArticleContent(bid, filename)
	if err != nil || content == nil {
		return
	}

	ar := article.NewRenderer()
	buf, err := ar.Render(content)
	if err == nil {
		a.ParsedTitle = ar.ParsedTitle()
		a.PreviewContent = ar.PreviewContent()
		a.ContentHtml = buf.Bytes()
		a.IsValid = true
	}
	return
}

func hasPermViewBoard(brd pttbbs.Board) error {
	if !pttbbs.IsValidBrdName(brd.BrdName) || brd.Over18 || brd.Hidden {
		return fmt.Errorf("No permission: %s", brd.BrdName)
	}
	return nil
}

func boardlist(ptt pttbbs.Pttbbs, indent string, root int, loop map[int]bool) {
	if loop[root] {
		//fmt.Println(indent, "loop skipped")
		return
	}
	loop[root] = true

	children, err := ptt.GetBoardChildren(root)
	if err != nil {
		fmt.Print(err)
		return
	}

	for _, bid := range children {
		if brd, err := ptt.GetBoard(bid); err == nil {
			if !brd.Hidden {
				//fmt.Println(indent, bid, brd)
				if !brd.IsBoard {
					boardlist(ptt, indent+"  ", bid, loop)
				} else {
					fmt.Println(brd.BrdName)
				}
			}
		} else {
			//fmt.Println(indent, bid, err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import "testing"

func TestUntouched(t *testing.T) {
	data := []struct {
		given, created, removed, untouched int64
	}{
		{
			given:     -1000,
			created:   2,
			removed:   10,
			untouched: -1,
		},
		{
			given:     0,
			created:   0,
			removed:   0,
			untouched: -2,
		},
		{
			given:     5,
			created:   3,
			removed:   0,
			untouched: 2,
		},
		{
			given:     10,
			created:   0,
			removed:   0,
			untouched: 10,
		},
		{
			given:     100,
			created:   100,
			removed:   100,
			untouched: 0,
		},
		{
			given:     5,
			created:   0,
			removed:   100,
			untouched: 5,
		},
	}

	for _, d := range data {
		u := untouched(d.given, d.created, d.removed)
		if u != d.untouched {
			t.Errorf("wrong value, expected %d but got %d", d.untouched, u)
		}
	}
}
<commit_msg>helper test extra case<commit_after>package main

import "testing"

func TestUntouched(t *testing.T) {
	data := []struct {
		given, created, removed, untouched int64
	}{
		{
			given:     -1000,
			created:   2,
			removed:   10,
			untouched: -1,
		},
		{
			given:     0,
			created:   0,
			removed:   0,
			untouched: -2,
		},
		{
			given:     5,
			created:   3,
			removed:   0,
			untouched: 2,
		},
		{
			given:     10,
			created:   0,
			removed:   0,
			untouched: 10,
		},
		{
			given:     100,
			created:   100,
			removed:   100,
			untouched: 0,
		},
		{
			given:     5,
			created:   0,
			removed:   100,
			untouched: 5,
		},
		{
			given:     5,
			created:   6,
			removed:   0,
			untouched: 0,
		},
	}

	for _, d := range data {
		u := untouched(d.given, d.created, d.removed)
		if u != d.untouched {
			t.Errorf("wrong value, expected %d but got %d", d.untouched, u)
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2015 All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"os"
	"time"

	"github.com/codegangsta/cli"
	"github.com/coreos/go-oidc/jose"
	"fmt"
	"strings"
)

func main() {
	app := cli.NewApp()
	app.Version = "v0.0.1"
	app.Author = "Rohith Jayawardene"
	app.Email = "gambol99@gmail.com"
	app.Flags = []cli.Flag {
		cli.StringFlag{
			Name:	"issuer",
			Usage: 	"the issuer of the token when generate",
			Value:  "http://keycloak.example.com/auth/realms/test",
		},
		cli.StringFlag{
			Name:	"audience",
			Usage:  "the audience of the token is directed to",
			Value:  "test",
		},
		cli.StringSliceFlag{
			Name:   "realm-role",
			Usage:  "a list of roles the token includes from the realm",
		},
		cli.StringSliceFlag{
			Name:   "role",
			Usage:  "a list of client roles the role includes, i.e. NAME:ROLE",
		},
		cli.DurationFlag{
			Name:   "expiration",
			Usage:  "the time from now the token should expire",
			Value:  time.Duration(1) * time.Hour,
		},
		cli.StringSliceFlag{
			Name:   "claims",
			Usage:  "a series of keypair claims which should be added to the token",
		},
	}
	app.Action = func(cx *cli.Context) error {

		header := jose.JOSEHeader{
			"alg": "RS256",
		}

		claims := jose.Claims{
			"exp":		time.Now().Add(cx.Duration("expiration")).Unix(),
			"iat":          "1450372669",
			"iss":          cx.String("issuer"),
			"aud":          cx.String("audience"),
			"sub":          cx.String("subject"),
		}
		// step: add the realm roles if any
		if len(cx.StringSlice("realm-role")) {
			claims["realm_access"] = map[string]interface{}{"roles":[]string{}}
			for _, x := range cx.StringSlice("realm-roles") {
				claims["realm_access"]["roles"] = append(claims["realm_access"]["roles"], x)
			}
		}
		// step: add the roles if any
		if len(cx.StringSlice("role")) {
			claims["resource_access"] = map[string]interface{}{}
			for _, k := range cx.StringSlice("roles") {
				elements := strings.Split(k, ":")
				if len(elements) {

				}
				roles, found := claims["resource_access"][elements[0]]
				if !found {
					claims["resource_access"][elements[0]] = map[string]interface{}{
						"roles": []string{},
					}
					roles = claims["resource_access"][elements[0]]
				}
				for _, x := range cx.StringSlice("role") {
					roles["roles"] = append(roles["roles"], x)
				}
			}
		}
		// step: add the custom claims
		for k, v := range cx.StringSlice("claim") {

		}

		// step: create the token
		token, err := jose.NewJWT(header, claims)
		if err != nil {

		}

		// step: sign the token
		signer, err :=jose.NewSignerRSA()
		if err != nil {

		}

		signer.Sign(token.Encode())

		fmt.Fprintf(os.Stdout, token.Encode() + "\n")
	}

	app.Run(os.Args)
}

<commit_msg>Test CLI (#210)<commit_after><|endoftext|>


<commit_before>//
// Copyright 2017, Sander van Harmelen
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package gitlab

import (
	"fmt"
	"time"
)

// ProjectMirrorService handles communication with the project mirror
// related methods of the GitLab API.
//
// GitLAb API docs:
// https://docs.gitlab.com/ce/api/remote_mirrors.html
type ProjectMirrorService struct {
	client *Client
}

// ProjectMirror represents a project mirror configuration.
//
// GitLAb API docs:
// https://docs.gitlab.com/ce/api/remote_mirrors.html
type ProjectMirror struct {
	Enabled                *bool      `url:"enabled" json:"enabled"`
	ID                     *int       `url:"id" json:"id"`
	LastError              *string    `url:"last_error" json:"last_error"`
	LastSuccessfulUpdateAt *time.Time `url:"last_successful_update_at" json:"last_successful_update_at"`
	LastUpdateAt           *time.Time `url:"last_update_at" json:"last_update_at"`
	LastUpdateStartedAt    *time.Time `url:"last_update_started_at" json:"last_update_started_at"`
	OnlyProtectedBranches  *bool      `url:"only_protected_branches" json:"only_protected_branches"`
	KeepDivergentRefs      *bool      `url:"keep_divergent_refs" json:"keep_divergent_refs"`
	UpdateStatus           *string    `url:"update_status" json:"update_status"`
	URL                    *string    `url:"url" json:"url"`
}

// ListProjectMirror gets a list of mirrors configured on the project.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/remote_mirrors.html#list-a-projects-remote-mirrors
func (s *ProjectMirrorService) ListProjectMirror(pid interface{}, options ...RequestOptionFunc) ([]*ProjectMirror, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/remote_mirrors", pathEscape(project))

	req, err := s.client.NewRequest("GET", u, nil, options)
	if err != nil {
		return nil, nil, err
	}

	var pm []*ProjectMirror
	resp, err := s.client.Do(req, &pm)
	if err != nil {
		return nil, resp, err
	}

	return pm, resp, err

}

// AddProjectMirrorOptions contains the properties requires to create
// a new project mirror.
type AddProjectMirrorOptions struct {
	URL                   string `json:"url"`
	Enabled               bool   `json:"enabled"`
	OnlyProtectedBranches bool   `json:"only_protected_branches"`
	KeepDivergentRefs     bool   `json:"keep_divergent_refs"`
}

// AddProjectMirror creates a new mirror on the project.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/remote_mirrors.html#create-a-remote-mirror
func (s *ProjectMirrorService) AddProjectMirror(pid interface{}, opt *AddProjectMirrorOptions, options ...RequestOptionFunc) (*ProjectMirror, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/remote_mirrors", pathEscape(project))

	req, err := s.client.NewRequest("POST", u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	pm := new(ProjectMirror)
	resp, err := s.client.Do(req, pm)
	if err != nil {
		return nil, resp, err
	}

	return pm, resp, err
}

// EditProjectMirrorOptions contains the properties requires to edit
// an existing project mirror.
type EditProjectMirrorOptions struct {
	URL                   string `json:"url"`
	Enabled               bool   `json:"enabled"`
	OnlyProtectedBranches bool   `json:"only_protected_branches"`
	KeepDivergentRefs     bool   `json:"keep_divergent_refs"`
}

// EditProjectMirror updates a project team member to a specified access level..
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/remote_mirrors.html#update-a-remote-mirrors-attributes
func (s *ProjectMirrorService) EditProjectMirror(pid interface{}, mirror int, opt *EditProjectMirrorOptions, options ...RequestOptionFunc) (*ProjectMirror, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/remote_mirrors/%d", pathEscape(project), mirror)

	req, err := s.client.NewRequest("PUT", u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	pm := new(ProjectMirror)
	resp, err := s.client.Do(req, pm)
	if err != nil {
		return nil, resp, err
	}

	return pm, resp, err
}
<commit_msg>Tweak the code a little<commit_after>//
// Copyright 2017, Sander van Harmelen
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package gitlab

import (
	"fmt"
	"time"
)

// ProjectMirrorService handles communication with the project mirror
// related methods of the GitLab API.
//
// GitLAb API docs: https://docs.gitlab.com/ce/api/remote_mirrors.html
type ProjectMirrorService struct {
	client *Client
}

// ProjectMirror represents a project mirror configuration.
//
// GitLAb API docs: https://docs.gitlab.com/ce/api/remote_mirrors.html
type ProjectMirror struct {
	Enabled                bool       `json:"enabled"`
	ID                     int        `json:"id"`
	LastError              string     `json:"last_error"`
	LastSuccessfulUpdateAt *time.Time `json:"last_successful_update_at"`
	LastUpdateAt           *time.Time `json:"last_update_at"`
	LastUpdateStartedAt    *time.Time `json:"last_update_started_at"`
	OnlyProtectedBranches  bool       `json:"only_protected_branches"`
	KeepDivergentRefs      bool       `json:"keep_divergent_refs"`
	UpdateStatus           string     `json:"update_status"`
	URL                    string     `json:"url"`
}

// ListProjectMirror gets a list of mirrors configured on the project.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/remote_mirrors.html#list-a-projects-remote-mirrors
func (s *ProjectMirrorService) ListProjectMirror(pid interface{}, options ...RequestOptionFunc) ([]*ProjectMirror, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/remote_mirrors", pathEscape(project))

	req, err := s.client.NewRequest("GET", u, nil, options)
	if err != nil {
		return nil, nil, err
	}

	var pm []*ProjectMirror
	resp, err := s.client.Do(req, &pm)
	if err != nil {
		return nil, resp, err
	}

	return pm, resp, err

}

// AddProjectMirrorOptions contains the properties requires to create
// a new project mirror.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/remote_mirrors.html#create-a-remote-mirror
type AddProjectMirrorOptions struct {
	URL                   *string `url:"url,omitempty" json:"url,omitempty"`
	Enabled               *bool   `url:"enabled,omitempty" json:"enabled,omitempty"`
	OnlyProtectedBranches *bool   `url:"only_protected_branches,omitempty" json:"only_protected_branches,omitempty"`
	KeepDivergentRefs     *bool   `url:"keep_divergent_refs,omitempty" json:"keep_divergent_refs,omitempty"`
}

// AddProjectMirror creates a new mirror on the project.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/remote_mirrors.html#create-a-remote-mirror
func (s *ProjectMirrorService) AddProjectMirror(pid interface{}, opt *AddProjectMirrorOptions, options ...RequestOptionFunc) (*ProjectMirror, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/remote_mirrors", pathEscape(project))

	req, err := s.client.NewRequest("POST", u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	pm := new(ProjectMirror)
	resp, err := s.client.Do(req, pm)
	if err != nil {
		return nil, resp, err
	}

	return pm, resp, err
}

// EditProjectMirrorOptions contains the properties requires to edit
// an existing project mirror.
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/remote_mirrors.html#update-a-remote-mirrors-attributes
type EditProjectMirrorOptions struct {
	Enabled               *bool `url:"enabled,omitempty" json:"enabled,omitempty"`
	OnlyProtectedBranches *bool `url:"only_protected_branches,omitempty" json:"only_protected_branches,omitempty"`
	KeepDivergentRefs     *bool `url:"keep_divergent_refs,omitempty" json:"keep_divergent_refs,omitempty"`
}

// EditProjectMirror updates a project team member to a specified access level..
//
// GitLab API docs:
// https://docs.gitlab.com/ce/api/remote_mirrors.html#update-a-remote-mirrors-attributes
func (s *ProjectMirrorService) EditProjectMirror(pid interface{}, mirror int, opt *EditProjectMirrorOptions, options ...RequestOptionFunc) (*ProjectMirror, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/remote_mirrors/%d", pathEscape(project), mirror)

	req, err := s.client.NewRequest("PUT", u, opt, options)
	if err != nil {
		return nil, nil, err
	}

	pm := new(ProjectMirror)
	resp, err := s.client.Do(req, pm)
	if err != nil {
		return nil, resp, err
	}

	return pm, resp, err
}
<|endoftext|>


<commit_before>// Copyright 2014 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package prometheus is the core instrumentation package. It provides metrics
// primitives to instrument code for monitoring. It also offers a registry for
// metrics. Sub-packages allow to expose the registered metrics via HTTP
// (package promhttp) or push them to a Pushgateway (package push). There is
// also a sub-package promauto, which provides metrics constructors with
// automatic registration.
//
// All exported functions and methods are safe to be used concurrently unless
// specified otherwise.
//
// A Basic Example
//
// As a starting point, a very basic usage example:
//
//    package main
//
//    import (
//    	"log"
//    	"net/http"
//
//    	"github.com/prometheus/client_golang/prometheus"
//    	"github.com/prometheus/client_golang/prometheus/promhttp"
//    )
//
//    var (
//    	cpuTemp = prometheus.NewGauge(prometheus.GaugeOpts{
//    		Name: "cpu_temperature_celsius",
//    		Help: "Current temperature of the CPU.",
//    	})
//    	hdFailures = prometheus.NewCounterVec(
//    		prometheus.CounterOpts{
//    			Name: "hd_errors_total",
//    			Help: "Number of hard-disk errors.",
//    		},
//    		[]string{"device"},
//    	)
//    )
//
//    func init() {
//    	// Metrics have to be registered to be exposed:
//    	prometheus.MustRegister(cpuTemp)
//    	prometheus.MustRegister(hdFailures)
//    }
//
//    func main() {
//    	cpuTemp.Set(65.3)
//    	hdFailures.With(prometheus.Labels{"device":"/dev/sda"}).Inc()
//
//    	// The Handler function provides a default handler to expose metrics
//    	// via an HTTP server. "/metrics" is the usual endpoint for that.
//    	http.Handle("/metrics", promhttp.Handler())
//    	log.Fatal(http.ListenAndServe(":8080", nil))
//    }
//
//
// This is a complete program that exports two metrics, a Gauge and a Counter,
// the latter with a label attached to turn it into a (one-dimensional) vector.
//
// Metrics
//
// The number of exported identifiers in this package might appear a bit
// overwhelming. However, in addition to the basic plumbing shown in the example
// above, you only need to understand the different metric types and their
// vector versions for basic usage. Furthermore, if you are not concerned with
// fine-grained control of when and how to register metrics with the registry,
// have a look at the promauto package, which will effectively allow you to
// ignore registration altogether in simple cases.
//
// Above, you have already touched the Counter and the Gauge. There are two more
// advanced metric types: the Summary and Histogram. A more thorough description
// of those four metric types can be found in the Prometheus docs:
// https://prometheus.io/docs/concepts/metric_types/
//
// A fifth "type" of metric is Untyped. It behaves like a Gauge, but signals the
// Prometheus server not to assume anything about its type.
//
// In addition to the fundamental metric types Gauge, Counter, Summary,
// Histogram, and Untyped, a very important part of the Prometheus data model is
// the partitioning of samples along dimensions called labels, which results in
// metric vectors. The fundamental types are GaugeVec, CounterVec, SummaryVec,
// HistogramVec, and UntypedVec.
//
// While only the fundamental metric types implement the Metric interface, both
// the metrics and their vector versions implement the Collector interface. A
// Collector manages the collection of a number of Metrics, but for convenience,
// a Metric can also “collect itself”. Note that Gauge, Counter, Summary,
// Histogram, and Untyped are interfaces themselves while GaugeVec, CounterVec,
// SummaryVec, HistogramVec, and UntypedVec are not.
//
// To create instances of Metrics and their vector versions, you need a suitable
// …Opts struct, i.e. GaugeOpts, CounterOpts, SummaryOpts, HistogramOpts, or
// UntypedOpts.
//
// Custom Collectors and constant Metrics
//
// While you could create your own implementations of Metric, most likely you
// will only ever implement the Collector interface on your own. At a first
// glance, a custom Collector seems handy to bundle Metrics for common
// registration (with the prime example of the different metric vectors above,
// which bundle all the metrics of the same name but with different labels).
//
// There is a more involved use case, too: If you already have metrics
// available, created outside of the Prometheus context, you don't need the
// interface of the various Metric types. You essentially want to mirror the
// existing numbers into Prometheus Metrics during collection. An own
// implementation of the Collector interface is perfect for that. You can create
// Metric instances “on the fly” using NewConstMetric, NewConstHistogram, and
// NewConstSummary (and their respective Must… versions). That will happen in
// the Collect method. The Describe method has to return separate Desc
// instances, representative of the “throw-away” metrics to be created later.
// NewDesc comes in handy to create those Desc instances. Alternatively, you
// could return no Desc at all, which will marke the Collector “unchecked”.  No
// checks are porformed at registration time, but metric consistency will still
// be ensured at scrape time, i.e. any inconsistencies will lead to scrape
// errors. Thus, with unchecked Collectors, the responsibility to not collect
// metrics that lead to inconsistencies in the total scrape result lies with the
// implementer of the Collector. While this is not a desirable state, it is
// sometimes necessary. The typical use case is a situation where the exact
// metrics to be returned by a Collector cannot be predicted at registration
// time, but the implementer has sufficient knowledge of the whole system to
// guarantee metric consistency.
//
// The Collector example illustrates the use case. You can also look at the
// source code of the processCollector (mirroring process metrics), the
// goCollector (mirroring Go metrics), or the expvarCollector (mirroring expvar
// metrics) as examples that are used in this package itself.
//
// If you just need to call a function to get a single float value to collect as
// a metric, GaugeFunc, CounterFunc, or UntypedFunc might be interesting
// shortcuts.
//
// Advanced Uses of the Registry
//
// While MustRegister is the by far most common way of registering a Collector,
// sometimes you might want to handle the errors the registration might cause.
// As suggested by the name, MustRegister panics if an error occurs. With the
// Register function, the error is returned and can be handled.
//
// An error is returned if the registered Collector is incompatible or
// inconsistent with already registered metrics. The registry aims for
// consistency of the collected metrics according to the Prometheus data model.
// Inconsistencies are ideally detected at registration time, not at collect
// time. The former will usually be detected at start-up time of a program,
// while the latter will only happen at scrape time, possibly not even on the
// first scrape if the inconsistency only becomes relevant later. That is the
// main reason why a Collector and a Metric have to describe themselves to the
// registry.
//
// So far, everything we did operated on the so-called default registry, as it
// can be found in the global DefaultRegisterer variable. With NewRegistry, you
// can create a custom registry, or you can even implement the Registerer or
// Gatherer interfaces yourself. The methods Register and Unregister work in the
// same way on a custom registry as the global functions Register and Unregister
// on the default registry.
//
// There are a number of uses for custom registries: You can use registries with
// special properties, see NewPedanticRegistry. You can avoid global state, as
// it is imposed by the DefaultRegisterer. You can use multiple registries at
// the same time to expose different metrics in different ways.  You can use
// separate registries for testing purposes.
//
// Also note that the DefaultRegisterer comes registered with a Collector for Go
// runtime metrics (via NewGoCollector) and a Collector for process metrics (via
// NewProcessCollector). With a custom registry, you are in control and decide
// yourself about the Collectors to register.
//
// HTTP Exposition
//
// The Registry implements the Gatherer interface. The caller of the Gather
// method can then expose the gathered metrics in some way. Usually, the metrics
// are served via HTTP on the /metrics endpoint. That's happening in the example
// above. The tools to expose metrics via HTTP are in the promhttp sub-package.
// (The top-level functions in the prometheus package are deprecated.)
//
// Pushing to the Pushgateway
//
// Function for pushing to the Pushgateway can be found in the push sub-package.
//
// Graphite Bridge
//
// Functions and examples to push metrics from a Gatherer to Graphite can be
// found in the graphite sub-package.
//
// Other Means of Exposition
//
// More ways of exposing metrics can easily be added by following the approaches
// of the existing implementations.
package prometheus
<commit_msg>Fix some spelling errors (#523)<commit_after>// Copyright 2014 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package prometheus is the core instrumentation package. It provides metrics
// primitives to instrument code for monitoring. It also offers a registry for
// metrics. Sub-packages allow to expose the registered metrics via HTTP
// (package promhttp) or push them to a Pushgateway (package push). There is
// also a sub-package promauto, which provides metrics constructors with
// automatic registration.
//
// All exported functions and methods are safe to be used concurrently unless
// specified otherwise.
//
// A Basic Example
//
// As a starting point, a very basic usage example:
//
//    package main
//
//    import (
//    	"log"
//    	"net/http"
//
//    	"github.com/prometheus/client_golang/prometheus"
//    	"github.com/prometheus/client_golang/prometheus/promhttp"
//    )
//
//    var (
//    	cpuTemp = prometheus.NewGauge(prometheus.GaugeOpts{
//    		Name: "cpu_temperature_celsius",
//    		Help: "Current temperature of the CPU.",
//    	})
//    	hdFailures = prometheus.NewCounterVec(
//    		prometheus.CounterOpts{
//    			Name: "hd_errors_total",
//    			Help: "Number of hard-disk errors.",
//    		},
//    		[]string{"device"},
//    	)
//    )
//
//    func init() {
//    	// Metrics have to be registered to be exposed:
//    	prometheus.MustRegister(cpuTemp)
//    	prometheus.MustRegister(hdFailures)
//    }
//
//    func main() {
//    	cpuTemp.Set(65.3)
//    	hdFailures.With(prometheus.Labels{"device":"/dev/sda"}).Inc()
//
//    	// The Handler function provides a default handler to expose metrics
//    	// via an HTTP server. "/metrics" is the usual endpoint for that.
//    	http.Handle("/metrics", promhttp.Handler())
//    	log.Fatal(http.ListenAndServe(":8080", nil))
//    }
//
//
// This is a complete program that exports two metrics, a Gauge and a Counter,
// the latter with a label attached to turn it into a (one-dimensional) vector.
//
// Metrics
//
// The number of exported identifiers in this package might appear a bit
// overwhelming. However, in addition to the basic plumbing shown in the example
// above, you only need to understand the different metric types and their
// vector versions for basic usage. Furthermore, if you are not concerned with
// fine-grained control of when and how to register metrics with the registry,
// have a look at the promauto package, which will effectively allow you to
// ignore registration altogether in simple cases.
//
// Above, you have already touched the Counter and the Gauge. There are two more
// advanced metric types: the Summary and Histogram. A more thorough description
// of those four metric types can be found in the Prometheus docs:
// https://prometheus.io/docs/concepts/metric_types/
//
// A fifth "type" of metric is Untyped. It behaves like a Gauge, but signals the
// Prometheus server not to assume anything about its type.
//
// In addition to the fundamental metric types Gauge, Counter, Summary,
// Histogram, and Untyped, a very important part of the Prometheus data model is
// the partitioning of samples along dimensions called labels, which results in
// metric vectors. The fundamental types are GaugeVec, CounterVec, SummaryVec,
// HistogramVec, and UntypedVec.
//
// While only the fundamental metric types implement the Metric interface, both
// the metrics and their vector versions implement the Collector interface. A
// Collector manages the collection of a number of Metrics, but for convenience,
// a Metric can also “collect itself”. Note that Gauge, Counter, Summary,
// Histogram, and Untyped are interfaces themselves while GaugeVec, CounterVec,
// SummaryVec, HistogramVec, and UntypedVec are not.
//
// To create instances of Metrics and their vector versions, you need a suitable
// …Opts struct, i.e. GaugeOpts, CounterOpts, SummaryOpts, HistogramOpts, or
// UntypedOpts.
//
// Custom Collectors and constant Metrics
//
// While you could create your own implementations of Metric, most likely you
// will only ever implement the Collector interface on your own. At a first
// glance, a custom Collector seems handy to bundle Metrics for common
// registration (with the prime example of the different metric vectors above,
// which bundle all the metrics of the same name but with different labels).
//
// There is a more involved use case, too: If you already have metrics
// available, created outside of the Prometheus context, you don't need the
// interface of the various Metric types. You essentially want to mirror the
// existing numbers into Prometheus Metrics during collection. An own
// implementation of the Collector interface is perfect for that. You can create
// Metric instances “on the fly” using NewConstMetric, NewConstHistogram, and
// NewConstSummary (and their respective Must… versions). That will happen in
// the Collect method. The Describe method has to return separate Desc
// instances, representative of the “throw-away” metrics to be created later.
// NewDesc comes in handy to create those Desc instances. Alternatively, you
// could return no Desc at all, which will mark the Collector “unchecked”.  No
// checks are performed at registration time, but metric consistency will still
// be ensured at scrape time, i.e. any inconsistencies will lead to scrape
// errors. Thus, with unchecked Collectors, the responsibility to not collect
// metrics that lead to inconsistencies in the total scrape result lies with the
// implementer of the Collector. While this is not a desirable state, it is
// sometimes necessary. The typical use case is a situation where the exact
// metrics to be returned by a Collector cannot be predicted at registration
// time, but the implementer has sufficient knowledge of the whole system to
// guarantee metric consistency.
//
// The Collector example illustrates the use case. You can also look at the
// source code of the processCollector (mirroring process metrics), the
// goCollector (mirroring Go metrics), or the expvarCollector (mirroring expvar
// metrics) as examples that are used in this package itself.
//
// If you just need to call a function to get a single float value to collect as
// a metric, GaugeFunc, CounterFunc, or UntypedFunc might be interesting
// shortcuts.
//
// Advanced Uses of the Registry
//
// While MustRegister is the by far most common way of registering a Collector,
// sometimes you might want to handle the errors the registration might cause.
// As suggested by the name, MustRegister panics if an error occurs. With the
// Register function, the error is returned and can be handled.
//
// An error is returned if the registered Collector is incompatible or
// inconsistent with already registered metrics. The registry aims for
// consistency of the collected metrics according to the Prometheus data model.
// Inconsistencies are ideally detected at registration time, not at collect
// time. The former will usually be detected at start-up time of a program,
// while the latter will only happen at scrape time, possibly not even on the
// first scrape if the inconsistency only becomes relevant later. That is the
// main reason why a Collector and a Metric have to describe themselves to the
// registry.
//
// So far, everything we did operated on the so-called default registry, as it
// can be found in the global DefaultRegisterer variable. With NewRegistry, you
// can create a custom registry, or you can even implement the Registerer or
// Gatherer interfaces yourself. The methods Register and Unregister work in the
// same way on a custom registry as the global functions Register and Unregister
// on the default registry.
//
// There are a number of uses for custom registries: You can use registries with
// special properties, see NewPedanticRegistry. You can avoid global state, as
// it is imposed by the DefaultRegisterer. You can use multiple registries at
// the same time to expose different metrics in different ways.  You can use
// separate registries for testing purposes.
//
// Also note that the DefaultRegisterer comes registered with a Collector for Go
// runtime metrics (via NewGoCollector) and a Collector for process metrics (via
// NewProcessCollector). With a custom registry, you are in control and decide
// yourself about the Collectors to register.
//
// HTTP Exposition
//
// The Registry implements the Gatherer interface. The caller of the Gather
// method can then expose the gathered metrics in some way. Usually, the metrics
// are served via HTTP on the /metrics endpoint. That's happening in the example
// above. The tools to expose metrics via HTTP are in the promhttp sub-package.
// (The top-level functions in the prometheus package are deprecated.)
//
// Pushing to the Pushgateway
//
// Function for pushing to the Pushgateway can be found in the push sub-package.
//
// Graphite Bridge
//
// Functions and examples to push metrics from a Gatherer to Graphite can be
// found in the graphite sub-package.
//
// Other Means of Exposition
//
// More ways of exposing metrics can easily be added by following the approaches
// of the existing implementations.
package prometheus
<|endoftext|>