<commit_before>/*
Copyright 2021 The Ceph-CSI Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"errors"
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"

	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
)

var (
	totalRequiredReviews int
	retry                int
)

type retestConfig struct {
	retryLimit          string
	requiredReviewCount string
	exemptlabel         string
	requiredlabel       string
	githubToken         string
	owner               string
	repo                string
	client              *github.Client
}

// standard input env variables in action more details at
// https://docs.github.com/en/actions/creating-actions/metadata-syntax-for-github-actions#inputs
func getConfig() *retestConfig {
	c := &retestConfig{}
	c.retryLimit = os.Getenv("INPUT_MAX-RETRY")
	c.requiredReviewCount = os.Getenv("INPUT_REQUIRED-APPROVE-COUNT")
	c.exemptlabel = os.Getenv("INPUT_EXEMPT-LABEL")
	c.requiredlabel = os.Getenv("INPUT_REQUIRED-LABEL")
	c.githubToken = os.Getenv("GITHUB_TOKEN")
	c.owner, c.repo = func() (string, string) {
		if os.Getenv("GITHUB_REPOSITORY") != "" {
			if len(strings.Split(os.Getenv("GITHUB_REPOSITORY"), "/")) == 2 {
				return strings.Split(os.Getenv("GITHUB_REPOSITORY"), "/")[0], strings.Split(os.Getenv("GITHUB_REPOSITORY"), "/")[1]
			}
		}
		return "", ""
	}()
	return c
}

// validate validates the input parameters.
func (c retestConfig) validate() error {
	if c.requiredlabel == "" {
		return errors.New("required-label is not set")
	}

	if c.githubToken == "" {
		return errors.New("GITHUB_TOKEN is not set")
	}

	if c.owner == "" || c.repo == "" {
		return errors.New("GITHUB_REPOSITORY is not set")
	}

	return nil
}

// createClient creates a new secure client.
func (c *retestConfig) createClient() {
	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: c.githubToken},
	)
	tc := oauth2.NewClient(context.TODO(), ts)
	c.client = github.NewClient(tc)
}

func main() {
	var err error
	c := getConfig()
	if err = c.validate(); err != nil {
		log.Fatalf("%v", err)
	}

	retry, err = strconv.Atoi(c.retryLimit)
	if err != nil {
		log.Fatalf("max-retry %q is not valid", c.retryLimit)
	}

	totalRequiredReviews, err = strconv.Atoi(c.requiredReviewCount)
	if err != nil {
		log.Fatalf("required-review-count %q is not valid", c.requiredReviewCount)
	}

	c.createClient()

	opt := &github.PullRequestListOptions{}
	req, _, err := c.client.PullRequests.List(context.TODO(), c.owner, c.repo, opt)
	if err != nil {
		log.Fatalf("failed to list pull requests %v\n", err)
	}
	for _, re := range req {
		if *re.State == "open" {
			prNumber := re.GetNumber()
			log.Printf("PR with ID %d with Title %q is open\n", prNumber, re.GetTitle())
			for _, l := range re.Labels {
				// check if label is exempt
				if strings.EqualFold(c.exemptlabel, l.GetName()) {
					continue
				}
				// check if label is matching
				if !strings.EqualFold(c.requiredlabel, l.GetName()) {
					continue
				}

				// check if PR has required approvals
				if !c.checkPRRequiredApproval(prNumber) {
					continue
				}

				log.Printf("checking status for PR %d with label %s", prNumber, l.GetName())
				rs, _, err := c.client.Repositories.ListStatuses(context.TODO(), c.owner, c.repo, re.GetHead().GetSHA(), &github.ListOptions{})
				if err != nil {
					log.Printf("failed to list status %v\n", err)
					continue
				}

				statusList := filterStatusList(rs)
				failedTestFound := false
				for _, r := range statusList {
					log.Printf("found context %s with status %s\n", r.GetContext(), r.GetState())
					if contains([]string{"failed", "failure"}, r.GetState()) {
						log.Printf("found failed test %s\n", r.GetContext())
						failedTestFound = true
						// rebase the pr if it is behind the devel branch.
						if *re.MergeableState == "BEHIND" {
							comment := &github.IssueComment{
								Body: github.String("@mergifyio rebase"),
							}
							_, _, err := c.client.Issues.CreateComment(context.TODO(), c.owner, c.repo, prNumber, comment)
							if err != nil {
								log.Printf("failed to create comment %v\n", err)
							}
							break
						}

						// check if retest limit is reached
						msg := fmt.Sprintf("/retest %s", r.GetContext())
						ok, err := c.checkRetestLimitReached(prNumber, msg)
						if err != nil {
							log.Printf("failed to check retest limit %v\n", err)
							continue
						}
						if ok {
							log.Printf("Pull Request %d: %q reached  maximum attempt. skipping retest %v\n", prNumber, r.GetContext(), retry)
							continue
						}

						comment := &github.IssueComment{
							Body: github.String(msg),
						}
						_, _, err = c.client.Issues.CreateComment(context.TODO(), c.owner, c.repo, prNumber, comment)
						if err != nil {
							log.Printf("failed to create comment %v\n", err)
							continue
						}
						// Post comment with target URL for retesting
						msg = fmt.Sprintf("@%s %q test failed. Logs are available at [location](%s) for debugging", re.GetUser().GetLogin(), r.GetContext(), r.GetTargetURL())
						comment.Body = github.String(msg)
						_, _, err = c.client.Issues.CreateComment(context.TODO(), c.owner, c.repo, prNumber, comment)
						if err != nil {
							log.Printf("failed to create comment %v\n", err)
							continue
						}
					}
				}

				if failedTestFound {
					// comment `@Mergifyio requeue` so mergifyio adds the pr back into the queue.
					msg := "@Mergifyio requeue"
					comment := &github.IssueComment{
						Body: github.String(msg),
					}
					_, _, err = c.client.Issues.CreateComment(context.TODO(), c.owner, c.repo, prNumber, comment)
					if err != nil {
						log.Printf("failed to create comment %q: %v\n", msg, err)
					}
					// exit after adding retests to a pr once to avoid retesting multiple prs
					// at the same time.
					break
				}
			}
		}
	}
}

// checkPRRequiredApproval check PullRequest has required approvals.
func (c *retestConfig) checkPRRequiredApproval(prNumber int) bool {
	opts := github.ListOptions{PerPage: 100} // defaults to 30 reviews, too few sometimes
	rev, _, err := c.client.PullRequests.ListReviews(context.TODO(), c.owner, c.repo, prNumber, &opts)
	if err != nil {
		log.Printf("failed to list reviews %v\n", err)
		return false
	}
	approvedReviews := 0
	for _, rv := range rev {
		if rv.GetState() == "APPROVED" {
			approvedReviews += 1
		}
	}
	if !(approvedReviews >= totalRequiredReviews) {
		log.Printf("total approved reviews for PR %d are %d but required %d", prNumber, approvedReviews, totalRequiredReviews)
		return false
	}

	return true
}

// checkRetestLimitReached check if retest limit is reached.
func (c *retestConfig) checkRetestLimitReached(prNumber int, msg string) (bool, error) {
	creq, _, err := c.client.Issues.ListComments(context.TODO(), c.owner, c.repo, prNumber, &github.IssueListCommentsOptions{})
	if err != nil {
		return false, err
	}
	retestCount := 0

	for _, pc := range creq {
		if pc.GetBody() == msg {
			retestCount += 1
		}
	}
	log.Printf("found %d retries and remaining %d retries\n", retestCount, retry-retestCount)
	if retestCount >= int(retry) {
		return true, nil
	}

	return false, nil
}

// containers check if slice contains string.
func contains(s []string, e string) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}

	return false
}

// filterStatusesList returns list of unique and recently updated github RepoStatuses.
// Raw github RepoStatus list may contain duplicate and older statuses.
func filterStatusList(rawStatusList []*github.RepoStatus) []*github.RepoStatus {
	testStatus := make(map[string]*github.RepoStatus)

	for _, r := range rawStatusList {
		status, ok := testStatus[r.GetContext()]
		if !ok || r.GetUpdatedAt().After(status.GetUpdatedAt()) {
			testStatus[r.GetContext()] = r
		}
	}

	statusList := make([]*github.RepoStatus, 0)
	for _, rs := range testStatus {
		statusList = append(statusList, rs)
	}

	return statusList
}
<commit_msg>ci: prevent panic in retest action on `nil` strings<commit_after>/*
Copyright 2021 The Ceph-CSI Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"context"
	"errors"
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"

	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
)

var (
	totalRequiredReviews int
	retry                int
)

type retestConfig struct {
	retryLimit          string
	requiredReviewCount string
	exemptlabel         string
	requiredlabel       string
	githubToken         string
	owner               string
	repo                string
	client              *github.Client
}

// standard input env variables in action more details at
// https://docs.github.com/en/actions/creating-actions/metadata-syntax-for-github-actions#inputs
func getConfig() *retestConfig {
	c := &retestConfig{}
	c.retryLimit = os.Getenv("INPUT_MAX-RETRY")
	c.requiredReviewCount = os.Getenv("INPUT_REQUIRED-APPROVE-COUNT")
	c.exemptlabel = os.Getenv("INPUT_EXEMPT-LABEL")
	c.requiredlabel = os.Getenv("INPUT_REQUIRED-LABEL")
	c.githubToken = os.Getenv("GITHUB_TOKEN")
	c.owner, c.repo = func() (string, string) {
		if os.Getenv("GITHUB_REPOSITORY") != "" {
			if len(strings.Split(os.Getenv("GITHUB_REPOSITORY"), "/")) == 2 {
				return strings.Split(os.Getenv("GITHUB_REPOSITORY"), "/")[0], strings.Split(os.Getenv("GITHUB_REPOSITORY"), "/")[1]
			}
		}
		return "", ""
	}()
	return c
}

// validate validates the input parameters.
func (c retestConfig) validate() error {
	if c.requiredlabel == "" {
		return errors.New("required-label is not set")
	}

	if c.githubToken == "" {
		return errors.New("GITHUB_TOKEN is not set")
	}

	if c.owner == "" || c.repo == "" {
		return errors.New("GITHUB_REPOSITORY is not set")
	}

	return nil
}

// createClient creates a new secure client.
func (c *retestConfig) createClient() {
	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: c.githubToken},
	)
	tc := oauth2.NewClient(context.TODO(), ts)
	c.client = github.NewClient(tc)
}

func main() {
	var err error
	c := getConfig()
	if err = c.validate(); err != nil {
		log.Fatalf("%v", err)
	}

	retry, err = strconv.Atoi(c.retryLimit)
	if err != nil {
		log.Fatalf("max-retry %q is not valid", c.retryLimit)
	}

	totalRequiredReviews, err = strconv.Atoi(c.requiredReviewCount)
	if err != nil {
		log.Fatalf("required-review-count %q is not valid", c.requiredReviewCount)
	}

	c.createClient()

	opt := &github.PullRequestListOptions{}
	req, _, err := c.client.PullRequests.List(context.TODO(), c.owner, c.repo, opt)
	if err != nil {
		log.Fatalf("failed to list pull requests %v\n", err)
	}
	for _, re := range req {
		if (re.State != nil) && (*re.State == "open") {
			prNumber := re.GetNumber()
			log.Printf("PR with ID %d with Title %q is open\n", prNumber, re.GetTitle())
			for _, l := range re.Labels {
				// check if label is exempt
				if strings.EqualFold(c.exemptlabel, l.GetName()) {
					continue
				}
				// check if label is matching
				if !strings.EqualFold(c.requiredlabel, l.GetName()) {
					continue
				}

				// check if PR has required approvals
				if !c.checkPRRequiredApproval(prNumber) {
					continue
				}

				log.Printf("checking status for PR %d with label %s", prNumber, l.GetName())
				rs, _, err := c.client.Repositories.ListStatuses(context.TODO(), c.owner, c.repo, re.GetHead().GetSHA(), &github.ListOptions{})
				if err != nil {
					log.Printf("failed to list status %v\n", err)
					continue
				}

				statusList := filterStatusList(rs)
				failedTestFound := false
				for _, r := range statusList {
					log.Printf("found context %s with status %s\n", r.GetContext(), r.GetState())
					if contains([]string{"failed", "failure"}, r.GetState()) {
						log.Printf("found failed test %s\n", r.GetContext())
						failedTestFound = true
						// rebase the pr if it is behind the devel branch.
						if (re.MergeableState != nil) && (*re.MergeableState == "BEHIND") {
							comment := &github.IssueComment{
								Body: github.String("@mergifyio rebase"),
							}
							_, _, err := c.client.Issues.CreateComment(context.TODO(), c.owner, c.repo, prNumber, comment)
							if err != nil {
								log.Printf("failed to create comment %v\n", err)
							}
							break
						}

						// check if retest limit is reached
						msg := fmt.Sprintf("/retest %s", r.GetContext())
						ok, err := c.checkRetestLimitReached(prNumber, msg)
						if err != nil {
							log.Printf("failed to check retest limit %v\n", err)
							continue
						}
						if ok {
							log.Printf("Pull Request %d: %q reached  maximum attempt. skipping retest %v\n", prNumber, r.GetContext(), retry)
							continue
						}

						comment := &github.IssueComment{
							Body: github.String(msg),
						}
						_, _, err = c.client.Issues.CreateComment(context.TODO(), c.owner, c.repo, prNumber, comment)
						if err != nil {
							log.Printf("failed to create comment %v\n", err)
							continue
						}
						// Post comment with target URL for retesting
						msg = fmt.Sprintf("@%s %q test failed. Logs are available at [location](%s) for debugging", re.GetUser().GetLogin(), r.GetContext(), r.GetTargetURL())
						comment.Body = github.String(msg)
						_, _, err = c.client.Issues.CreateComment(context.TODO(), c.owner, c.repo, prNumber, comment)
						if err != nil {
							log.Printf("failed to create comment %v\n", err)
							continue
						}
					}
				}

				if failedTestFound {
					// comment `@Mergifyio requeue` so mergifyio adds the pr back into the queue.
					msg := "@Mergifyio requeue"
					comment := &github.IssueComment{
						Body: github.String(msg),
					}
					_, _, err = c.client.Issues.CreateComment(context.TODO(), c.owner, c.repo, prNumber, comment)
					if err != nil {
						log.Printf("failed to create comment %q: %v\n", msg, err)
					}
					// exit after adding retests to a pr once to avoid retesting multiple prs
					// at the same time.
					break
				}
			}
		}
	}
}

// checkPRRequiredApproval check PullRequest has required approvals.
func (c *retestConfig) checkPRRequiredApproval(prNumber int) bool {
	opts := github.ListOptions{PerPage: 100} // defaults to 30 reviews, too few sometimes
	rev, _, err := c.client.PullRequests.ListReviews(context.TODO(), c.owner, c.repo, prNumber, &opts)
	if err != nil {
		log.Printf("failed to list reviews %v\n", err)
		return false
	}
	approvedReviews := 0
	for _, rv := range rev {
		if rv.GetState() == "APPROVED" {
			approvedReviews += 1
		}
	}
	if !(approvedReviews >= totalRequiredReviews) {
		log.Printf("total approved reviews for PR %d are %d but required %d", prNumber, approvedReviews, totalRequiredReviews)
		return false
	}

	return true
}

// checkRetestLimitReached check if retest limit is reached.
func (c *retestConfig) checkRetestLimitReached(prNumber int, msg string) (bool, error) {
	creq, _, err := c.client.Issues.ListComments(context.TODO(), c.owner, c.repo, prNumber, &github.IssueListCommentsOptions{})
	if err != nil {
		return false, err
	}
	retestCount := 0

	for _, pc := range creq {
		if pc.GetBody() == msg {
			retestCount += 1
		}
	}
	log.Printf("found %d retries and remaining %d retries\n", retestCount, retry-retestCount)
	if retestCount >= int(retry) {
		return true, nil
	}

	return false, nil
}

// containers check if slice contains string.
func contains(s []string, e string) bool {
	for _, a := range s {
		if a == e {
			return true
		}
	}

	return false
}

// filterStatusesList returns list of unique and recently updated github RepoStatuses.
// Raw github RepoStatus list may contain duplicate and older statuses.
func filterStatusList(rawStatusList []*github.RepoStatus) []*github.RepoStatus {
	testStatus := make(map[string]*github.RepoStatus)

	for _, r := range rawStatusList {
		status, ok := testStatus[r.GetContext()]
		if !ok || r.GetUpdatedAt().After(status.GetUpdatedAt()) {
			testStatus[r.GetContext()] = r
		}
	}

	statusList := make([]*github.RepoStatus, 0)
	for _, rs := range testStatus {
		statusList = append(statusList, rs)
	}

	return statusList
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package serviceaccount

import (
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	coreinformers "k8s.io/client-go/informers/core/v1"
	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
	"k8s.io/klog"

	"github.com/kubernetes-sigs/multi-tenancy/incubator/virtualcluster/pkg/syncer/constants"
	ctrl "github.com/kubernetes-sigs/multi-tenancy/incubator/virtualcluster/pkg/syncer/controller"
	sc "github.com/kubernetes-sigs/multi-tenancy/incubator/virtualcluster/pkg/syncer/controller"
	"github.com/kubernetes-sigs/multi-tenancy/incubator/virtualcluster/pkg/syncer/conversion"
	"github.com/kubernetes-sigs/multi-tenancy/incubator/virtualcluster/pkg/syncer/manager"
	"github.com/kubernetes-sigs/multi-tenancy/incubator/virtualcluster/pkg/syncer/reconciler"
)

type controller struct {
	client                               v1core.CoreV1Interface
	saInformer                           coreinformers.ServiceAccountInformer
	multiClusterServiceAccountController *sc.MultiClusterController
}

func Register(
	client v1core.CoreV1Interface,
	saInformer coreinformers.ServiceAccountInformer,
	controllerManager *manager.ControllerManager,
) {
	c := &controller{
		client: client,
	}

	// Create the multi cluster secret controller
	options := sc.Options{Reconciler: c}
	multiClusterSecretController, err := sc.NewController("tenant-masters-service-account-controller", &v1.ServiceAccount{}, options)
	if err != nil {
		klog.Errorf("failed to create multi cluster secret controller %v", err)
		return
	}
	c.multiClusterServiceAccountController = multiClusterSecretController

	controllerManager.AddController(c)
}

func (c *controller) StartUWS(stopCh <-chan struct{}) error {
	return nil
}

func (c *controller) StartDWS(stopCh <-chan struct{}) error {
	return c.multiClusterServiceAccountController.Start(stopCh)
}

func (c *controller) Reconcile(request reconciler.Request) (reconciler.Result, error) {
	klog.Infof("reconcile service account %s/%s %s event for cluster %s", request.Namespace, request.Name, request.Event, request.Cluster.Name)

	switch request.Event {
	case reconciler.AddEvent:
		err := c.reconcileServiceAccountCreate(request.Cluster.Name, request.Namespace, request.Name, request.Obj.(*v1.ServiceAccount))
		if err != nil {
			klog.Errorf("failed reconcile service account %s/%s CREATE of cluster %s %v", request.Namespace, request.Name, request.Cluster.Name, err)
			return reconciler.Result{Requeue: true}, err
		}
	case reconciler.UpdateEvent:
		err := c.reconcileServiceAccountUpdate(request.Cluster.Name, request.Namespace, request.Name, request.Obj.(*v1.ServiceAccount))
		if err != nil {
			klog.Errorf("failed reconcile service account %s/%s CREATE of cluster %s %v", request.Namespace, request.Name, request.Cluster.Name, err)
			return reconciler.Result{Requeue: true}, err
		}
	case reconciler.DeleteEvent:
		err := c.reconcileServiceAccountRemove(request.Cluster.Name, request.Namespace, request.Name, request.Obj.(*v1.ServiceAccount))
		if err != nil {
			klog.Errorf("failed reconcile service account %s/%s DELETE of cluster %s %v", request.Namespace, request.Name, request.Cluster.Name, err)
			return reconciler.Result{Requeue: true}, err
		}
	}
	return reconciler.Result{}, nil
}

func (c *controller) reconcileServiceAccountCreate(cluster, namespace, name string, secret *v1.ServiceAccount) error {
	targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace)
	// just mark the default service account as a tenant related resource.
	// service account controller will create it for us.
	if name == "default" {
		sa, err := c.client.ServiceAccounts(targetNamespace).Get("default", metav1.GetOptions{})
		if err != nil {
			// maybe the sa is not created, retry
			return err
		}

		if len(sa.Annotations) == 0 {
			sa.Annotations = make(map[string]string)
		}
		sa.Annotations[constants.LabelCluster] = cluster
		_, err = c.client.ServiceAccounts(targetNamespace).Update(sa)
		return err
	}
	newObj, err := conversion.BuildMetadata(cluster, targetNamespace, secret)
	if err != nil {
		return err
	}

	pServiceAccount := newObj.(*v1.ServiceAccount)
	// set to empty and token controller will regenerate one.
	pServiceAccount.Secrets = nil

	_, err = c.client.ServiceAccounts(targetNamespace).Create(pServiceAccount)
	if errors.IsAlreadyExists(err) {
		klog.Infof("service account %s/%s of cluster %s already exist in super master", namespace, name, cluster)
		return nil
	}
	return err
}

func (c *controller) reconcileServiceAccountUpdate(cluster, namespace, name string, secret *v1.ServiceAccount) error {
	// do nothing.
	return nil
}

func (c *controller) reconcileServiceAccountRemove(cluster, namespace, name string, secret *v1.ServiceAccount) error {
	targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace)
	opts := &metav1.DeleteOptions{
		PropagationPolicy: &constants.DefaultDeletionPolicy,
	}
	err := c.client.ServiceAccounts(targetNamespace).Delete(name, opts)
	if errors.IsNotFound(err) {
		klog.Warningf("service account %s/%s of cluster not found in super master", namespace, name, cluster)
		return nil
	}
	return err
}

func (c *controller) AddCluster(cluster ctrl.ClusterInterface) {
	klog.Infof("tenant-masters-service-account-controller watch cluster %s for serviceaccount resource", cluster.GetClusterName())
	err := c.multiClusterServiceAccountController.WatchClusterResource(cluster, sc.WatchOptions{})
	if err != nil {
		klog.Errorf("failed to watch cluster %s secret event: %v", cluster.GetClusterName(), err)
	}
}

func (c *controller) RemoveCluster(cluster ctrl.ClusterInterface) {
	klog.Infof("tenant-masters-service-account-controller stop watching cluster %s for serviceaccount resource", cluster.GetClusterName())
	c.multiClusterServiceAccountController.TeardownClusterResource(cluster)
}
<commit_msg>Fix a build error<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package serviceaccount

import (
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	coreinformers "k8s.io/client-go/informers/core/v1"
	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
	"k8s.io/klog"

	"github.com/kubernetes-sigs/multi-tenancy/incubator/virtualcluster/pkg/syncer/constants"
	ctrl "github.com/kubernetes-sigs/multi-tenancy/incubator/virtualcluster/pkg/syncer/controller"
	sc "github.com/kubernetes-sigs/multi-tenancy/incubator/virtualcluster/pkg/syncer/controller"
	"github.com/kubernetes-sigs/multi-tenancy/incubator/virtualcluster/pkg/syncer/conversion"
	"github.com/kubernetes-sigs/multi-tenancy/incubator/virtualcluster/pkg/syncer/manager"
	"github.com/kubernetes-sigs/multi-tenancy/incubator/virtualcluster/pkg/syncer/reconciler"
)

type controller struct {
	client                               v1core.CoreV1Interface
	saInformer                           coreinformers.ServiceAccountInformer
	multiClusterServiceAccountController *sc.MultiClusterController
}

func Register(
	client v1core.CoreV1Interface,
	saInformer coreinformers.ServiceAccountInformer,
	controllerManager *manager.ControllerManager,
) {
	c := &controller{
		client: client,
	}

	// Create the multi cluster secret controller
	options := sc.Options{Reconciler: c}
	multiClusterSecretController, err := sc.NewController("tenant-masters-service-account-controller", &v1.ServiceAccount{}, options)
	if err != nil {
		klog.Errorf("failed to create multi cluster secret controller %v", err)
		return
	}
	c.multiClusterServiceAccountController = multiClusterSecretController

	controllerManager.AddController(c)
}

func (c *controller) StartUWS(stopCh <-chan struct{}) error {
	return nil
}

func (c *controller) StartDWS(stopCh <-chan struct{}) error {
	return c.multiClusterServiceAccountController.Start(stopCh)
}

func (c *controller) Reconcile(request reconciler.Request) (reconciler.Result, error) {
	klog.Infof("reconcile service account %s/%s %s event for cluster %s", request.Namespace, request.Name, request.Event, request.Cluster.Name)

	switch request.Event {
	case reconciler.AddEvent:
		err := c.reconcileServiceAccountCreate(request.Cluster.Name, request.Namespace, request.Name, request.Obj.(*v1.ServiceAccount))
		if err != nil {
			klog.Errorf("failed reconcile service account %s/%s CREATE of cluster %s %v", request.Namespace, request.Name, request.Cluster.Name, err)
			return reconciler.Result{Requeue: true}, err
		}
	case reconciler.UpdateEvent:
		err := c.reconcileServiceAccountUpdate(request.Cluster.Name, request.Namespace, request.Name, request.Obj.(*v1.ServiceAccount))
		if err != nil {
			klog.Errorf("failed reconcile service account %s/%s CREATE of cluster %s %v", request.Namespace, request.Name, request.Cluster.Name, err)
			return reconciler.Result{Requeue: true}, err
		}
	case reconciler.DeleteEvent:
		err := c.reconcileServiceAccountRemove(request.Cluster.Name, request.Namespace, request.Name, request.Obj.(*v1.ServiceAccount))
		if err != nil {
			klog.Errorf("failed reconcile service account %s/%s DELETE of cluster %s %v", request.Namespace, request.Name, request.Cluster.Name, err)
			return reconciler.Result{Requeue: true}, err
		}
	}
	return reconciler.Result{}, nil
}

func (c *controller) reconcileServiceAccountCreate(cluster, namespace, name string, secret *v1.ServiceAccount) error {
	targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace)
	// just mark the default service account as a tenant related resource.
	// service account controller will create it for us.
	if name == "default" {
		sa, err := c.client.ServiceAccounts(targetNamespace).Get("default", metav1.GetOptions{})
		if err != nil {
			// maybe the sa is not created, retry
			return err
		}

		if len(sa.Annotations) == 0 {
			sa.Annotations = make(map[string]string)
		}
		sa.Annotations[constants.LabelCluster] = cluster
		_, err = c.client.ServiceAccounts(targetNamespace).Update(sa)
		return err
	}
	newObj, err := conversion.BuildMetadata(cluster, targetNamespace, secret)
	if err != nil {
		return err
	}

	pServiceAccount := newObj.(*v1.ServiceAccount)
	// set to empty and token controller will regenerate one.
	pServiceAccount.Secrets = nil

	_, err = c.client.ServiceAccounts(targetNamespace).Create(pServiceAccount)
	if errors.IsAlreadyExists(err) {
		klog.Infof("service account %s/%s of cluster %s already exist in super master", namespace, name, cluster)
		return nil
	}
	return err
}

func (c *controller) reconcileServiceAccountUpdate(cluster, namespace, name string, secret *v1.ServiceAccount) error {
	// do nothing.
	return nil
}

func (c *controller) reconcileServiceAccountRemove(cluster, namespace, name string, secret *v1.ServiceAccount) error {
	targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace)
	opts := &metav1.DeleteOptions{
		PropagationPolicy: &constants.DefaultDeletionPolicy,
	}
	err := c.client.ServiceAccounts(targetNamespace).Delete(name, opts)
	if errors.IsNotFound(err) {
		klog.Warningf("service account %s/%s of cluster %s not found in super master", namespace, name, cluster)
		return nil
	}
	return err
}

func (c *controller) AddCluster(cluster ctrl.ClusterInterface) {
	klog.Infof("tenant-masters-service-account-controller watch cluster %s for serviceaccount resource", cluster.GetClusterName())
	err := c.multiClusterServiceAccountController.WatchClusterResource(cluster, sc.WatchOptions{})
	if err != nil {
		klog.Errorf("failed to watch cluster %s secret event: %v", cluster.GetClusterName(), err)
	}
}

func (c *controller) RemoveCluster(cluster ctrl.ClusterInterface) {
	klog.Infof("tenant-masters-service-account-controller stop watching cluster %s for serviceaccount resource", cluster.GetClusterName())
	c.multiClusterServiceAccountController.TeardownClusterResource(cluster)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/fatih/color"
	"github.com/joho/godotenv"
)

var envFiles = []string{
	".env.local",
	".env",
}

const (
	usage   = `usage: run <command> [<args>...]`
	logFile = ".run.log"
)

func exitErr(err error) {
	fmt.Fprintf(os.Stderr, "error occured: %v", err)
	fmt.Fprintln(os.Stderr)
	os.Exit(1)
}

func getEnv() map[string]string {

	for _, f := range envFiles {
		stat, err := os.Stat(f)
		if err == nil && !stat.IsDir() {
			m, err := godotenv.Read(f)
			if err != nil {
				err := fmt.Errorf("error loading env file: %w", err)
				log.Fatal(err)
			}
			return m
		}
	}
	return map[string]string{}
}

func openLogFile() (*os.File, error) {
	return os.OpenFile(logFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0644)
}

func prepareLogFile() error {
	l, err := openLogFile()
	if err != nil {
		return fmt.Errorf("error preparing log file: %w", err)
	}
	return l.Close()
}

func main() {
	// we only need the time to see changes in output, date not that important
	log.SetFlags(log.Ltime)

	envVars := getEnv()
	args := os.Args[1:]
	if len(args) == 0 {
		exitErr(fmt.Errorf("args missing"))
	}

	if err := prepareLogFile(); err != nil {
		exitErr(fmt.Errorf("error preparing logs: %v", err))
	}

	if args[0] == "log" {
		fmt.Fprintln(os.Stderr, "to view the logs, run the following command or $(run log)")
		fmt.Printf("tail -f %s", logFile)
		fmt.Println()
		return
	}

	cmd, cancel, err := run(args, envVars)
	if err != nil {
		err := fmt.Errorf("error running command: %w", err)
		exitErr(err)
	}

	for {
		kill := func() error {
			return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
		}
		start := func() {
			cmd, cancel, err = run(args, envVars)
			if err != nil {
				err := fmt.Errorf("error running command: %w", err)
				fmt.Fprintln(os.Stderr, err)
			}
		}

		continueOnFailure := false

		fmt.Println()
		fmt.Println("input one of the following")
		fmt.Println("  'r' to hard restart")
		fmt.Println("  'ro' to soft restart")
		fmt.Println("  'x' to terminate")
		fmt.Println()
		fmt.Print("  input: ")

		var line string
		fmt.Scanln(&line)

		// extra newline for log clarity
		fmt.Println()

		switch line {
		case "r":
			continueOnFailure = true
			fallthrough
		case "ro":
			log.Println("restarting...")
		case "x":
			log.Println("terminating...")
			kill()
			cancel()
			os.Exit(0)
		default:
			log.Printf("unrecognized input '%s'", line)
			log.Println()
			continue
		}

		if cmd == nil {
			log.Println("command not running, cannot restart")
			start()
			continue
		}

		// attempt to kill process and all it's children
		log.Println("attempting to kill process with pid", cmd.Process.Pid)
		if err := kill(); err != nil {
			err := fmt.Errorf("error terminating process: %w", err)
			fmt.Fprintln(os.Stderr, err)

			if continueOnFailure {
				start()
			}
			continue
		}

		log.Println("process killed.")
		log.Println()

		// cancel the process context
		cancel()

		// let's wait for the process in case there's some delay in quitting
		cmd.Process.Wait()

		// let's attempt to run the program again
		start()
	}
}

func run(args []string, vars map[string]string) (*exec.Cmd, func(), error) {
	// convert into shell script for sh to run
	sh := ""
	for _, a := range args {
		sh += " " + strconv.Quote(a)
	}

	// sha args
	cmdArgs := []string{"-c", sh}

	// use a cancel context to be on safe side
	ctx, cancel := context.WithCancel(context.Background())
	cmd := exec.CommandContext(ctx, "sh", cmdArgs...)

	// environment variables
	cmd.Env = os.Environ()
	for k, v := range vars {
		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v))
	}

	// ensure we can kill the children
	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}

	// set log outputs
	out, err := openLogFile()
	if err != nil {
		return nil, cancel, fmt.Errorf("error opening log file: %w", err)
	}

	cmd.Stdout = &lineWriter{prefix: color.New(color.BgBlue, color.FgWhite).Sprint("stdout"), out: out}
	cmd.Stderr = &lineWriter{prefix: color.New(color.BgRed, color.FgWhite).Sprint("stderr"), out: out}

	log.Println("running {", strings.Join(args, " "), "}...")
	log.Println()

	if err := cmd.Start(); err != nil {
		return nil, cancel, fmt.Errorf("error starting command: %w", err)
	}

	logStartup(out)

	shutdown := func() {
		cancel()
		logShutdown(out)
		out.Close()
	}

	return cmd, shutdown, nil
}

func logStartup(out io.Writer) {
	fmt.Fprintln(out)
	fmt.Fprintln(out, "starting up at", time.Now())
	fmt.Fprintln(out)
}
func logShutdown(out io.Writer) {
	fmt.Fprintln(out)
	fmt.Fprintln(out, "shutting down at", time.Now())
	fmt.Fprintln(out)
}

var _ io.Writer = (*lineWriter)(nil)

// lineWriter is a simple writer that only writes to an underlying writer when
// a newline is encountered.
type lineWriter struct {
	out    io.Writer
	prefix string

	buf bytes.Buffer
	sync.Mutex
}

func (l *lineWriter) Write(b []byte) (int, error) {
	l.Lock()
	defer l.Unlock()

	for i := 0; i < len(b); i++ {

		// special case: replace escaped chars with their real value
		// newline, tab, quote, backslack
		if b[i] == '\\' {
			// peek if available
			if i+1 < len(b) {
				i++
				switch b[i] {
				case 'n':
					b[i] = '\n'
				case 't':
					b[i] = '\t'

				// do nothing for these, escape char already skipped
				case '\\':
				case '"':
				case '\'':

				// otherwise, don't skip escape char
				default:
					i--
				}
			}
		}

		// cache the char
		l.buf.WriteByte(b[i])

		// write to underlying writer if newline is encountered
		if b[i] == '\n' {
			l.out.Write([]byte(l.prefix + " "))
			l.buf.WriteTo(l.out)
			l.buf.Truncate(0)
		}
	}

	// all bytes are always successfully written
	return len(b), nil
}
<commit_msg>envrun: minor log improvement<commit_after>package main

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/fatih/color"
	"github.com/joho/godotenv"
)

var envFiles = []string{
	".env.local",
	".env",
}

const (
	usage   = `usage: run <command> [<args>...]`
	logFile = ".run.log"
)

func exitErr(err error) {
	fmt.Fprintf(os.Stderr, "error occured: %v", err)
	fmt.Fprintln(os.Stderr)
	os.Exit(1)
}

func getEnv() map[string]string {

	for _, f := range envFiles {
		stat, err := os.Stat(f)
		if err == nil && !stat.IsDir() {
			m, err := godotenv.Read(f)
			if err != nil {
				err := fmt.Errorf("error loading env file: %w", err)
				log.Fatal(err)
			}
			return m
		}
	}
	return map[string]string{}
}

func openLogFile() (*os.File, error) {
	return os.OpenFile(logFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0644)
}

func prepareLogFile() error {
	l, err := openLogFile()
	if err != nil {
		return fmt.Errorf("error preparing log file: %w", err)
	}
	return l.Close()
}

func main() {
	// we only need the time to see changes in output, date not that important
	log.SetFlags(log.Ltime)

	envVars := getEnv()
	args := os.Args[1:]
	if len(args) == 0 {
		exitErr(fmt.Errorf("args missing"))
	}

	if err := prepareLogFile(); err != nil {
		exitErr(fmt.Errorf("error preparing logs: %v", err))
	}

	if args[0] == "log" {
		fmt.Fprintln(os.Stderr, "to view the logs, run the following command or $(run log)")
		fmt.Printf("tail -f %s", logFile)
		fmt.Println()
		return
	}

	cmd, cancel, err := run(args, envVars)
	if err != nil {
		err := fmt.Errorf("error running command: %w", err)
		exitErr(err)
	}

	for {
		kill := func() error {
			return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
		}
		start := func() {
			cmd, cancel, err = run(args, envVars)
			if err != nil {
				err := fmt.Errorf("error running command: %w", err)
				fmt.Fprintln(os.Stderr, err)
			}
		}

		continueOnFailure := false

		fmt.Println()
		fmt.Println("input one of the following")
		fmt.Println("  'r' to hard restart")
		fmt.Println("  'ro' to soft restart")
		fmt.Println("  'x' to terminate")
		fmt.Println()
		fmt.Print("  input: ")

		var line string
		fmt.Scanln(&line)

		// extra newline for log clarity
		fmt.Println()

		switch line {
		case "r":
			continueOnFailure = true
			fallthrough
		case "ro":
			log.Println("restarting...")
		case "x":
			log.Println("terminating...")
			kill()
			cancel()
			os.Exit(0)
		default:
			log.Printf("unrecognized input '%s'", line)
			log.Println()
			continue
		}

		if cmd == nil {
			log.Println("command not running, cannot restart")
			start()
			continue
		}

		// attempt to kill process and all it's children
		log.Println("attempting to kill process with pid", cmd.Process.Pid)
		if err := kill(); err != nil {
			err := fmt.Errorf("error terminating process: %w", err)
			fmt.Fprintln(os.Stderr, err)

			if continueOnFailure {
				start()
			}
			continue
		}

		log.Println("process killed.")
		log.Println()

		// cancel the process context
		cancel()

		// let's wait for the process in case there's some delay in quitting
		cmd.Process.Wait()

		// let's attempt to run the program again
		start()
	}
}

func run(args []string, vars map[string]string) (*exec.Cmd, func(), error) {
	// convert into shell script for sh to run
	sh := ""
	for _, a := range args {
		sh += " " + strconv.Quote(a)
	}

	// sha args
	cmdArgs := []string{"-c", sh}

	// use a cancel context to be on safe side
	ctx, cancel := context.WithCancel(context.Background())
	cmd := exec.CommandContext(ctx, "sh", cmdArgs...)

	// environment variables
	cmd.Env = os.Environ()
	for k, v := range vars {
		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v))
	}

	// ensure we can kill the children
	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}

	// set log outputs
	out, err := openLogFile()
	if err != nil {
		return nil, cancel, fmt.Errorf("error opening log file: %w", err)
	}

	cmd.Stdout = &lineWriter{prefix: color.New(color.BgBlue, color.FgWhite).Sprint("stdout"), out: out}
	cmd.Stderr = &lineWriter{prefix: color.New(color.BgRed, color.FgWhite).Sprint("stderr"), out: out}

	log.Println("running {", strings.Join(args, ", "), "}...")
	log.Println()

	if err := cmd.Start(); err != nil {
		return nil, cancel, fmt.Errorf("error starting command: %w", err)
	}

	logStartup(out)

	shutdown := func() {
		cancel()
		logShutdown(out)
		out.Close()
	}

	return cmd, shutdown, nil
}

func logStartup(out io.Writer) {
	fmt.Fprintln(out)
	fmt.Fprintln(out, "starting up at", time.Now())
	fmt.Fprintln(out)
}
func logShutdown(out io.Writer) {
	fmt.Fprintln(out)
	fmt.Fprintln(out, "shutting down at", time.Now())
	fmt.Fprintln(out)
}

var _ io.Writer = (*lineWriter)(nil)

// lineWriter is a simple writer that only writes to an underlying writer when
// a newline is encountered.
type lineWriter struct {
	out    io.Writer
	prefix string

	buf bytes.Buffer
	sync.Mutex
}

func (l *lineWriter) Write(b []byte) (int, error) {
	l.Lock()
	defer l.Unlock()

	for i := 0; i < len(b); i++ {

		// special case: replace escaped chars with their real value
		// newline, tab, quote, backslack
		if b[i] == '\\' {
			// peek if available
			if i+1 < len(b) {
				i++
				switch b[i] {
				case 'n':
					b[i] = '\n'
				case 't':
					b[i] = '\t'

				// do nothing for these, escape char already skipped
				case '\\':
				case '"':
				case '\'':

				// otherwise, don't skip escape char
				default:
					i--
				}
			}
		}

		// cache the char
		l.buf.WriteByte(b[i])

		// write to underlying writer if newline is encountered
		if b[i] == '\n' {
			l.out.Write([]byte(l.prefix + " "))
			l.buf.WriteTo(l.out)
			l.buf.Truncate(0)
		}
	}

	// all bytes are always successfully written
	return len(b), nil
}
<|endoftext|>


<commit_before>package filesystemfake

import (
	"bytes"
	"errors"
	"os"
	"time"
)

var (
	ErrFileOperationNotPermitted = errors.New("fake file does not exist and therefore this operation is not permitted")
	ErrFileDoesNotExist          = errors.New("fake file does not exist")
)

// Fake represents are readable in memory version of os.File which
// can be used for testing.
type file struct {
	Name    string
	Mode    os.FileMode
	ModTime time.Time
	Buffer  *bytes.Reader
}

// newFile creates a new file instances based on a file name
// and it's contents from strings. The content of the new instance will
// be stored in an internal bytes.Reader instance. The default file mode
// will be 0777 and the last modification time the moment when the
// function is called.
func newFile(name string, content []byte) file {
	return file{
		Name:    name,
		Mode:    os.FileMode(0777),
		ModTime: time.Now(),
		Buffer:  bytes.NewReader(content),
	}
}

// Close wraps io.Closer's functionality and does no operation.
func (f file) Close() error {
	return nil
}

// Read wraps io.Reader's functionality around the internal
// bytes.Reader instance.
func (f file) Read(p []byte) (n int, err error) {
	return f.Buffer.Read(p)
}

// ReadAt wraps io.ReaderAt's functionality around the internalt
// bytes.Reader instance.
func (f file) ReadAt(p []byte, off int64) (n int, err error) {
	return f.Buffer.ReadAt(p, off)
}

// Seek wraps io.Seeker's functionality around the internalt
// bytes.Reader instance.
func (f file) Seek(offset int64, whence int) (int64, error) {
	return f.Buffer.Seek(offset, whence)
}

// Stat returns the fileInfo structure describing the file
// instance.
func (f file) Stat() (os.FileInfo, error) {
	return fileInfo{File: f}, nil
}

func newFileInfo(name string, content []byte) os.FileInfo {
	newFileInfo := fileInfo{
		File: newFile(name, content),
	}

	return newFileInfo
}

// fileInfo describes a wrapped file isntance and is returned
// by file.Stat
type fileInfo struct {
	File file
}

// Name returns the base name of the file instance.
func (fi fileInfo) Name() string {
	return fi.File.Name
}

// Size returns the length in bytes of the file's
// internal bytes.Reader instance.
func (fi fileInfo) Size() int64 {
	return int64(fi.File.Buffer.Len())
}

// Mode returns file mode bits of the file instance.
func (fi fileInfo) Mode() os.FileMode {
	return fi.File.Mode
}

// ModTime returns the modification time of the file instance.
func (fi fileInfo) ModTime() time.Time {
	return fi.File.ModTime
}

// IsDir always return false since it only uses file instances.
func (fi fileInfo) IsDir() bool {
	return false
}

// Sys always returns nil to stay conformant to the os.FileInfo interface.
func (fi fileInfo) Sys() interface{} {
	return nil
}
<commit_msg>removed dead code<commit_after>package filesystemfake

import (
	"bytes"
	"errors"
	"os"
	"time"
)

// Fake represents are readable in memory version of os.File which
// can be used for testing.
type file struct {
	Name    string
	Mode    os.FileMode
	ModTime time.Time
	Buffer  *bytes.Reader
}

// newFile creates a new file instances based on a file name
// and it's contents from strings. The content of the new instance will
// be stored in an internal bytes.Reader instance. The default file mode
// will be 0777 and the last modification time the moment when the
// function is called.
func newFile(name string, content []byte) file {
	return file{
		Name:    name,
		Mode:    os.FileMode(0777),
		ModTime: time.Now(),
		Buffer:  bytes.NewReader(content),
	}
}

// Close wraps io.Closer's functionality and does no operation.
func (f file) Close() error {
	return nil
}

// Read wraps io.Reader's functionality around the internal
// bytes.Reader instance.
func (f file) Read(p []byte) (n int, err error) {
	return f.Buffer.Read(p)
}

// ReadAt wraps io.ReaderAt's functionality around the internalt
// bytes.Reader instance.
func (f file) ReadAt(p []byte, off int64) (n int, err error) {
	return f.Buffer.ReadAt(p, off)
}

// Seek wraps io.Seeker's functionality around the internalt
// bytes.Reader instance.
func (f file) Seek(offset int64, whence int) (int64, error) {
	return f.Buffer.Seek(offset, whence)
}

// Stat returns the fileInfo structure describing the file
// instance.
func (f file) Stat() (os.FileInfo, error) {
	return fileInfo{File: f}, nil
}

func newFileInfo(name string, content []byte) os.FileInfo {
	newFileInfo := fileInfo{
		File: newFile(name, content),
	}

	return newFileInfo
}

// fileInfo describes a wrapped file isntance and is returned
// by file.Stat
type fileInfo struct {
	File file
}

// Name returns the base name of the file instance.
func (fi fileInfo) Name() string {
	return fi.File.Name
}

// Size returns the length in bytes of the file's
// internal bytes.Reader instance.
func (fi fileInfo) Size() int64 {
	return int64(fi.File.Buffer.Len())
}

// Mode returns file mode bits of the file instance.
func (fi fileInfo) Mode() os.FileMode {
	return fi.File.Mode
}

// ModTime returns the modification time of the file instance.
func (fi fileInfo) ModTime() time.Time {
	return fi.File.ModTime
}

// IsDir always return false since it only uses file instances.
func (fi fileInfo) IsDir() bool {
	return false
}

// Sys always returns nil to stay conformant to the os.FileInfo interface.
func (fi fileInfo) Sys() interface{} {
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"math"
	"regexp"
	"time"

	"github.com/elastic/libbeat/cfgfile"
	"github.com/elastic/libbeat/common"
	"github.com/elastic/libbeat/logp"
	"github.com/elastic/libbeat/publisher"
	"github.com/elastic/libbeat/beat"
)

type ProcsMap map[int]*Process

type Topbeat struct {
	isAlive      bool
	period       time.Duration
	procs        []string
	procsMap     ProcsMap
	lastCpuTimes *CpuTimes
	TbConfig     ConfigSettings
	events       chan common.MapStr
}

func (tb *Topbeat) Config(b *beat.Beat) error {

	err := cfgfile.Read(&tb.TbConfig)
	if err != nil {
		logp.Err("Error reading configuration file: %v", err)
		return err
	}

	if tb.TbConfig.Input.Period != nil {
		tb.period = time.Duration(*tb.TbConfig.Input.Period) * time.Second
	} else {
		tb.period = 1 * time.Second
	}
	if tb.TbConfig.Input.Procs != nil {
		tb.procs = *tb.TbConfig.Input.Procs
	} else {
		tb.procs = []string{".*"} //all processes
	}

	logp.Debug("topbeat", "Init toppbeat")
	logp.Debug("topbeat", "Follow processes %q\n", tb.procs)
	logp.Debug("topbeat", "Period %v\n", tb.period)

	return nil
}

func (tb *Topbeat) Setup(b *beat.Beat) error {

	tb.events = publisher.Publisher.Queue
	return nil
}

func (t *Topbeat) Run(b *beat.Beat) error {

	t.isAlive = true

	t.initProcStats()

	var err error

	for t.isAlive {
		time.Sleep(t.period)

		err = t.exportSystemStats()
		if err != nil {
			logp.Err("Error reading system stats: %v", err)
		}
		err = t.exportProcStats()
		if err != nil {
			logp.Err("Error reading proc stats: %v", err)
		}
		err = t.exportFileSystemStats()
		if err != nil {
			logp.Err("Error reading fs stats: %v", err)
		}
	}

	return err
}

func (tb *Topbeat) Cleanup(b *beat.Beat) error {
	return nil
}

func (t *Topbeat) Stop() {

	t.isAlive = false
}

func (t *Topbeat) initProcStats() {

	t.procsMap = make(ProcsMap)

	if len(t.procs) == 0 {
		return
	}

	pids, err := Pids()
	if err != nil {
		logp.Warn("Getting the list of pids: %v", err)
	}

	logp.Debug("topbeat", "Pids: %v\n", pids)

	for _, pid := range pids {
		process, err := GetProcess(pid)
		if err != nil {
			logp.Debug("topbeat", "Skip process %d: %v", pid, err)
			continue
		}
		t.procsMap[process.Pid] = process
	}
}

func (t *Topbeat) exportProcStats() error {

	if len(t.procs) == 0 {
		return nil
	}

	pids, err := Pids()
	if err != nil {
		logp.Warn("Getting the list of pids: %v", err)
		return err
	}

	for _, pid := range pids {
		process, err := GetProcess(pid)
		if err != nil {
			logp.Debug("topbeat", "Skip process %d: %v", pid, err)
			continue
		}

		if t.MatchProcess(process.Name) {

			t.addProcCpuPercentage(process)
			t.addProcMemPercentage(process, 0 /*read total mem usage */)

			t.procsMap[process.Pid] = process

			event := common.MapStr{
				"timestamp":  common.Time(time.Now()),
				"type":       "proc",
				"proc.pid":   process.Pid,
				"proc.ppid":  process.Ppid,
				"proc.name":  process.Name,
				"proc.state": process.State,
				"proc.mem":   process.Mem,
				"proc.cpu":   process.Cpu,
			}
			t.events <- event
		}
	}
	return nil
}

func (t *Topbeat) exportSystemStats() error {

	load_stat, err := GetSystemLoad()
	if err != nil {
		logp.Warn("Getting load statistics: %v", err)
		return err
	}
	cpu_stat, err := GetCpuTimes()
	if err != nil {
		logp.Warn("Getting cpu times: %v", err)
		return err
	}

	t.addCpuPercentage(cpu_stat)

	mem_stat, err := GetMemory()
	if err != nil {
		logp.Warn("Getting memory details: %v", err)
		return err
	}
	t.addMemPercentage(mem_stat)

	swap_stat, err := GetSwap()
	if err != nil {
		logp.Warn("Getting swap details: %v", err)
		return err
	}
	t.addMemPercentage(swap_stat)

	event := common.MapStr{
		"timestamp": common.Time(time.Now()),
		"type":      "system",
		"load":      load_stat,
		"cpu":       cpu_stat,
		"mem":       mem_stat,
		"swap":      swap_stat,
	}

	t.events <- event

	return nil
}

func (t *Topbeat) exportFileSystemStats() error {

	fss, err := GetFileSystemList()
	if err != nil {
		logp.Warn("Getting filesystem list: %v", err)
		return err
	}

	for _, fs := range fss {
		fs_stat, err := GetFileSystemStat(fs)
		if err != nil {
			logp.Debug("topbeat", "Skip filesystem %d: %v", fs_stat, err)
			continue
		}
		t.addFileSystemUsedPercentage(fs_stat)

		event := common.MapStr{
			"timestamp": common.Time(time.Now()),
			"type":      "filesystem",
			"fs":        fs_stat,
		}
		t.events <- event
	}

	return nil
}

func (t *Topbeat) MatchProcess(name string) bool {

	for _, reg := range t.procs {
		matched, _ := regexp.MatchString(reg, name)
		if matched {
			return true
		}
	}
	return false
}

func (t *Topbeat) addMemPercentage(m *MemStat) {

	if m.Total == 0 {
		return
	}

	perc := float64(100*m.Used) / float64(m.Total)
	m.UsedPercent = Round(perc, .5, 2)
}

func (t *Topbeat) addFileSystemUsedPercentage(f *FileSystemStat) {

	if f.Total == 0 {
		return
	}

	perc := float64(100*f.Used) / float64(f.Total)
	f.UsedPercent = Round(perc, .5, 2)
}

func (t *Topbeat) addCpuPercentage(t2 *CpuTimes) {

	t1 := t.lastCpuTimes

	if t1 != nil && t2 != nil {
		all_delta := t2.sum() - t1.sum()

		calculate := func(field2 uint64, field1 uint64) float64 {

			perc := 0.0
			delta := field2 - field1
			perc = float64(100*delta) / float64(all_delta)
			return Round(perc, .5, 2)
		}

		t2.UserPercent = calculate(t2.User, t1.User)
		t2.SystemPercent = calculate(t2.System, t1.System)
	}

	t.lastCpuTimes = t2

}

func (t *Topbeat) addProcMemPercentage(proc *Process, total_phymem uint64) {

	// total_phymem is set to a value greater than zero in tests

	if total_phymem == 0 {
		mem_stat, err := GetMemory()
		if err != nil {
			logp.Warn("Getting memory details: %v", err)
			return
		}
		total_phymem = mem_stat.Total
	}

	perc := (float64(proc.Mem.Rss) / float64(total_phymem)) * 100

	proc.Mem.RssPercent = Round(perc, .5, 2)
}

func (t *Topbeat) addProcCpuPercentage(proc *Process) {

	oproc, ok := t.procsMap[proc.Pid]
	if ok {

		delta_proc := (proc.Cpu.User - oproc.Cpu.User) + (proc.Cpu.System - oproc.Cpu.System)
		delta_time := proc.ctime.Sub(oproc.ctime).Nanoseconds() / 1e6 // in milliseconds
		perc := float64(delta_proc) / float64(delta_time) * 100

		t.procsMap[proc.Pid] = proc

		proc.Cpu.UserPercent = Round(perc, .5, 2)

	}
}

func Round(val float64, roundOn float64, places int) (newVal float64) {
	var round float64
	pow := math.Pow(10, float64(places))
	digit := pow * val
	_, div := math.Modf(digit)
	if div >= roundOn {
		round = math.Ceil(digit)
	} else {
		round = math.Floor(digit)
	}
	newVal = round / pow
	return
}

func (t *CpuTimes) sum() uint64 {
	return t.User + t.Nice + t.System + t.Idle + t.IOWait + t.Irq + t.SoftIrq + t.Steal
}
<commit_msg>Remove debug message<commit_after>package main

import (
	"math"
	"regexp"
	"time"

	"github.com/elastic/libbeat/beat"
	"github.com/elastic/libbeat/cfgfile"
	"github.com/elastic/libbeat/common"
	"github.com/elastic/libbeat/logp"
	"github.com/elastic/libbeat/publisher"
)

type ProcsMap map[int]*Process

type Topbeat struct {
	isAlive      bool
	period       time.Duration
	procs        []string
	procsMap     ProcsMap
	lastCpuTimes *CpuTimes
	TbConfig     ConfigSettings
	events       chan common.MapStr
}

func (tb *Topbeat) Config(b *beat.Beat) error {

	err := cfgfile.Read(&tb.TbConfig)
	if err != nil {
		logp.Err("Error reading configuration file: %v", err)
		return err
	}

	if tb.TbConfig.Input.Period != nil {
		tb.period = time.Duration(*tb.TbConfig.Input.Period) * time.Second
	} else {
		tb.period = 1 * time.Second
	}
	if tb.TbConfig.Input.Procs != nil {
		tb.procs = *tb.TbConfig.Input.Procs
	} else {
		tb.procs = []string{".*"} //all processes
	}

	logp.Debug("topbeat", "Init toppbeat")
	logp.Debug("topbeat", "Follow processes %q\n", tb.procs)
	logp.Debug("topbeat", "Period %v\n", tb.period)

	return nil
}

func (tb *Topbeat) Setup(b *beat.Beat) error {

	tb.events = publisher.Publisher.Queue
	return nil
}

func (t *Topbeat) Run(b *beat.Beat) error {

	t.isAlive = true

	t.initProcStats()

	var err error

	for t.isAlive {
		time.Sleep(t.period)

		err = t.exportSystemStats()
		if err != nil {
			logp.Err("Error reading system stats: %v", err)
		}
		err = t.exportProcStats()
		if err != nil {
			logp.Err("Error reading proc stats: %v", err)
		}
		err = t.exportFileSystemStats()
		if err != nil {
			logp.Err("Error reading fs stats: %v", err)
		}
	}

	return err
}

func (tb *Topbeat) Cleanup(b *beat.Beat) error {
	return nil
}

func (t *Topbeat) Stop() {

	t.isAlive = false
}

func (t *Topbeat) initProcStats() {

	t.procsMap = make(ProcsMap)

	if len(t.procs) == 0 {
		return
	}

	pids, err := Pids()
	if err != nil {
		logp.Warn("Getting the list of pids: %v", err)
	}

	for _, pid := range pids {
		process, err := GetProcess(pid)
		if err != nil {
			logp.Debug("topbeat", "Skip process %d: %v", pid, err)
			continue
		}
		t.procsMap[process.Pid] = process
	}
}

func (t *Topbeat) exportProcStats() error {

	if len(t.procs) == 0 {
		return nil
	}

	pids, err := Pids()
	if err != nil {
		logp.Warn("Getting the list of pids: %v", err)
		return err
	}

	for _, pid := range pids {
		process, err := GetProcess(pid)
		if err != nil {
			logp.Debug("topbeat", "Skip process %d: %v", pid, err)
			continue
		}

		if t.MatchProcess(process.Name) {

			t.addProcCpuPercentage(process)
			t.addProcMemPercentage(process, 0 /*read total mem usage */)

			t.procsMap[process.Pid] = process

			event := common.MapStr{
				"timestamp":  common.Time(time.Now()),
				"type":       "proc",
				"proc.pid":   process.Pid,
				"proc.ppid":  process.Ppid,
				"proc.name":  process.Name,
				"proc.state": process.State,
				"proc.mem":   process.Mem,
				"proc.cpu":   process.Cpu,
			}
			t.events <- event
		}
	}
	return nil
}

func (t *Topbeat) exportSystemStats() error {

	load_stat, err := GetSystemLoad()
	if err != nil {
		logp.Warn("Getting load statistics: %v", err)
		return err
	}
	cpu_stat, err := GetCpuTimes()
	if err != nil {
		logp.Warn("Getting cpu times: %v", err)
		return err
	}

	t.addCpuPercentage(cpu_stat)

	mem_stat, err := GetMemory()
	if err != nil {
		logp.Warn("Getting memory details: %v", err)
		return err
	}
	t.addMemPercentage(mem_stat)

	swap_stat, err := GetSwap()
	if err != nil {
		logp.Warn("Getting swap details: %v", err)
		return err
	}
	t.addMemPercentage(swap_stat)

	event := common.MapStr{
		"timestamp": common.Time(time.Now()),
		"type":      "system",
		"load":      load_stat,
		"cpu":       cpu_stat,
		"mem":       mem_stat,
		"swap":      swap_stat,
	}

	t.events <- event

	return nil
}

func (t *Topbeat) exportFileSystemStats() error {

	fss, err := GetFileSystemList()
	if err != nil {
		logp.Warn("Getting filesystem list: %v", err)
		return err
	}

	for _, fs := range fss {
		fs_stat, err := GetFileSystemStat(fs)
		if err != nil {
			logp.Debug("topbeat", "Skip filesystem %d: %v", fs_stat, err)
			continue
		}
		t.addFileSystemUsedPercentage(fs_stat)

		event := common.MapStr{
			"timestamp": common.Time(time.Now()),
			"type":      "filesystem",
			"fs":        fs_stat,
		}
		t.events <- event
	}

	return nil
}

func (t *Topbeat) MatchProcess(name string) bool {

	for _, reg := range t.procs {
		matched, _ := regexp.MatchString(reg, name)
		if matched {
			return true
		}
	}
	return false
}

func (t *Topbeat) addMemPercentage(m *MemStat) {

	if m.Total == 0 {
		return
	}

	perc := float64(100*m.Used) / float64(m.Total)
	m.UsedPercent = Round(perc, .5, 2)
}

func (t *Topbeat) addFileSystemUsedPercentage(f *FileSystemStat) {

	if f.Total == 0 {
		return
	}

	perc := float64(100*f.Used) / float64(f.Total)
	f.UsedPercent = Round(perc, .5, 2)
}

func (t *Topbeat) addCpuPercentage(t2 *CpuTimes) {

	t1 := t.lastCpuTimes

	if t1 != nil && t2 != nil {
		all_delta := t2.sum() - t1.sum()

		calculate := func(field2 uint64, field1 uint64) float64 {

			perc := 0.0
			delta := field2 - field1
			perc = float64(100*delta) / float64(all_delta)
			return Round(perc, .5, 2)
		}

		t2.UserPercent = calculate(t2.User, t1.User)
		t2.SystemPercent = calculate(t2.System, t1.System)
	}

	t.lastCpuTimes = t2

}

func (t *Topbeat) addProcMemPercentage(proc *Process, total_phymem uint64) {

	// in unit tests, total_phymem is set to a value greater than zero

	if total_phymem == 0 {
		mem_stat, err := GetMemory()
		if err != nil {
			logp.Warn("Getting memory details: %v", err)
			return
		}
		total_phymem = mem_stat.Total
	}

	perc := (float64(proc.Mem.Rss) / float64(total_phymem)) * 100

	proc.Mem.RssPercent = Round(perc, .5, 2)
}

func (t *Topbeat) addProcCpuPercentage(proc *Process) {

	oproc, ok := t.procsMap[proc.Pid]
	if ok {

		delta_proc := (proc.Cpu.User - oproc.Cpu.User) + (proc.Cpu.System - oproc.Cpu.System)
		delta_time := proc.ctime.Sub(oproc.ctime).Nanoseconds() / 1e6 // in milliseconds
		perc := float64(delta_proc) / float64(delta_time) * 100

		t.procsMap[proc.Pid] = proc

		proc.Cpu.UserPercent = Round(perc, .5, 2)

	}
}

func Round(val float64, roundOn float64, places int) (newVal float64) {
	var round float64
	pow := math.Pow(10, float64(places))
	digit := pow * val
	_, div := math.Modf(digit)
	if div >= roundOn {
		round = math.Ceil(digit)
	} else {
		round = math.Floor(digit)
	}
	newVal = round / pow
	return
}

func (t *CpuTimes) sum() uint64 {
	return t.User + t.Nice + t.System + t.Idle + t.IOWait + t.Irq + t.SoftIrq + t.Steal
}
<|endoftext|>


<commit_before>package rethink

import (
	"errors"
	"log"

	"github.com/gorethink/gorethink"
)

// Config database
type Config struct {
	Host    string
	Db      string
	MaxOpen int
}

var s *gorethink.Session

// Init connection
func Init() {
	log.Println("[database]", "starting...")

	log.Println("[database]", "started!")
}

// GetSession database
func GetSession() *gorethink.Session {
	return s
}

// Connect to database
func Connect(c Config) {
	// check credentials
	err := errors.New("db host or db must defined")
	if c.Host == "" || c.Db == "" {
		log.Fatalln(err)
	}

	s, err = gorethink.Connect(gorethink.ConnectOpts{
		Address:  c.Host,
		Database: c.Db,
		MaxOpen:  c.MaxOpen,
	})
	if err != nil {
		log.Fatalln("[database]", err)
	}

	// create if not exists
	gorethink.DBCreate(c.Db).Run(s)

	// enabling json tag as alternative on component Objects
	gorethink.SetTags("gorethink", "json")
}
<commit_msg>remove unsued funcs<commit_after>package rethink

import (
	"errors"
	"log"

	"github.com/gorethink/gorethink"
)

// Config database
type Config struct {
	Host    string
	Db      string
	MaxOpen int
}

var s *gorethink.Session

// GetSession database
func GetSession() *gorethink.Session {
	return s
}

// Connect to database
func Connect(c Config) {
	// check credentials
	err := errors.New("db host or db must defined")
	if c.Host == "" || c.Db == "" {
		log.Fatalln(err)
	}

	s, err = gorethink.Connect(gorethink.ConnectOpts{
		Address:  c.Host,
		Database: c.Db,
		MaxOpen:  c.MaxOpen,
	})
	if err != nil {
		log.Fatalln("[database]", err)
	}

	// create if not exists
	gorethink.DBCreate(c.Db).Run(s)

	// enabling json tag as alternative on component Objects
	gorethink.SetTags("gorethink", "json")
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"log"
	"net/http"
	"strconv"
	"time"

	"github.com/toomore/gogrs/tradingdays"
)

// Log is show viwer log.
func Log(req *http.Request) {
	log.Println(req.URL, req.UserAgent(), req.Form)
}

// Home is home page.
func Home(w http.ResponseWriter, req *http.Request) {
	w.Write([]byte("Hello World"))
	Log(req)
}

type tradeJSON struct {
	Date time.Time `json:"date"`
	Open bool      `json:"open"`
}

type errorJSON struct {
	Error string `json:"error"`
}

// TradeOpen is "./open" page.
func TradeOpen(w http.ResponseWriter, req *http.Request) {
	var jsonStr []byte
	data, err := strconv.ParseInt(req.FormValue("q"), 10, 64)

	if csvcachetime.InCache(defaultcachetime) != true {
		tradingdays.DownloadCSV(true)
		csvcachetime.Set()
		log.Println("DownloadCSV.")
	}

	if err != nil {
		jsonStr, _ = json.Marshal(&errorJSON{Error: "Wrong date format"})
	} else {
		date := time.Unix(data, 0)
		jsonStr, _ = json.Marshal(&tradeJSON{
			Date: date.UTC(),
			Open: tradingdays.IsOpen(date.Year(), date.Month(), date.Day(), date.Location())})
	}
	w.Header().Set("Content-Type", "application/json")
	w.Write(jsonStr)
	Log(req)
}

type cachetime struct {
	timestamp int64
}

func (c *cachetime) Set() {
	c.timestamp = time.Now().Unix()
}

func (c *cachetime) InCache(seconds int64) bool {
	result := (time.Now().Unix() - c.timestamp) <= seconds
	if result != true {
		c.timestamp = 0
	}
	return result
}

var csvcachetime cachetime

var httpport string
var defaultcachetime int64

func init() {
	tradingdays.DownloadCSV(true)
	log.Println("Init DownloadCSV.")
	csvcachetime.Set()
	flag.StringVar(&httpport, "http", ":59123", "HTTP service address (e.g., ':59123')")
	flag.Int64Var(&defaultcachetime, "csvcachetime", 21600, "CSV cache time.")
}

func main() {
	flag.Parse()
	log.Println("http:", httpport, "csvcachetime:", defaultcachetime)
	http.HandleFunc("/", Home)
	http.HandleFunc("/open", TradeOpen)
	log.Fatal(http.ListenAndServe(httpport, nil))
}
<commit_msg>Add more info in home page.<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"strconv"
	"time"

	"github.com/toomore/gogrs/tradingdays"
)

// Log is show viwer log.
func Log(req *http.Request) {
	log.Println(req.URL, req.UserAgent(), req.Form)
}

// Home is home page.
func Home(w http.ResponseWriter, req *http.Request) {
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	w.Write([]byte("查詢台灣股市是否開市
"))
	w.Write([]byte(fmt.Sprintf("範例
", time.Now().Unix())))
	w.Write([]byte("[Docs] [github]
"))

	Log(req)
}

type tradeJSON struct {
	Date time.Time `json:"date"`
	Open bool      `json:"open"`
}

type errorJSON struct {
	Error string `json:"error"`
}

// TradeOpen is "./open" page.
func TradeOpen(w http.ResponseWriter, req *http.Request) {
	var jsonStr []byte
	data, err := strconv.ParseInt(req.FormValue("q"), 10, 64)

	if csvcachetime.InCache(defaultcachetime) != true {
		tradingdays.DownloadCSV(true)
		csvcachetime.Set()
		log.Println("DownloadCSV.")
	}

	if err != nil {
		jsonStr, _ = json.Marshal(&errorJSON{Error: "Wrong date format"})
	} else {
		date := time.Unix(data, 0)
		jsonStr, _ = json.Marshal(&tradeJSON{
			Date: date.UTC(),
			Open: tradingdays.IsOpen(date.Year(), date.Month(), date.Day(), date.Location())})
	}
	w.Header().Set("Content-Type", "application/json")
	w.Write(jsonStr)
	Log(req)
}

type cachetime struct {
	timestamp int64
}

func (c *cachetime) Set() {
	c.timestamp = time.Now().Unix()
}

func (c *cachetime) InCache(seconds int64) bool {
	result := (time.Now().Unix() - c.timestamp) <= seconds
	if result != true {
		c.timestamp = 0
	}
	return result
}

var csvcachetime cachetime

var httpport string
var defaultcachetime int64

func init() {
	tradingdays.DownloadCSV(true)
	log.Println("Init DownloadCSV.")
	csvcachetime.Set()
	flag.StringVar(&httpport, "http", ":59123", "HTTP service address (e.g., ':59123')")
	flag.Int64Var(&defaultcachetime, "csvcachetime", 21600, "CSV cache time.")
}

func main() {
	flag.Parse()
	log.Println("http:", httpport, "csvcachetime:", defaultcachetime)
	http.HandleFunc("/", Home)
	http.HandleFunc("/open", TradeOpen)
	log.Fatal(http.ListenAndServe(httpport, nil))
}
<|endoftext|>


<commit_before>/*
	This file is part of LogHub.

	LogHub is free software: you can redistribute it and/or modify
	it under the terms of the GNU General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	LogHub is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU General Public License for more details.

	You should have received a copy of the GNU General Public License
	along with LogHub.  If not, see <http://www.gnu.org/licenses/>.
*/
package main

import (
	"os"
	"encoding/binary"
)

const defaultEntryBufferSize = 1024
const entryPayloadBase = 16
const timestampLength = 8
const entryHeaderLength = entryPayloadBase + timestampLength
const hopLength = 64

type logQuery struct {
	from        int64
	to     		int64
	minSeverity int
	maxSeverity int
	source      string
	result      chan *LogEntry
}

type logClose struct {
	ack chan bool
}

type LogFile struct {
	writeChan chan *LogEntry
	readChan  chan *logQuery
	closeChan chan *logClose
}

type entryHeader struct {
	payloadLength     int32
	prevPayloadLength int32
	hop               int32
	backHop           int32
	timestamp         int64
}

func (h *entryHeader) NextOffset() int64 {
	return int64(h.payloadLength) + int64(entryPayloadBase)
}

func (h *entryHeader) PrevOffset() int64 {
	return int64(h.prevPayloadLength) + int64(entryPayloadBase)
}

func OpenCreateLogFile(path string) (*LogFile, error) {
	if f, err := os.OpenFile(path, os.O_RDWR | os.O_CREATE, 0660); err == nil {
		log := &LogFile{ make(chan *LogEntry), make(chan *logQuery), make(chan *logClose) }

		go log.run(f)

		return log, nil
	} else {
		return nil, err
	}
}

func writeUtf8(buf []byte, str string) []byte {
	var varintBuf [binary.MaxVarintLen64]byte

	l := binary.PutUvarint(varintBuf[:], uint64(len(str)))
	buf = append(buf, varintBuf[:l]...)
	buf = append(buf, []byte(str)...)

	return buf
}

func writeEntry(buf []byte, ent *LogEntry, prevPayloadLen int32, backHop int32) ([]byte, int32) {
	for i := 0; i < entryPayloadBase; i++ {
		buf = append(buf, byte(0))
	}

	binary.BigEndian.PutUint32(buf[4:], uint32(prevPayloadLen))
	binary.BigEndian.PutUint32(buf[12:], uint32(backHop))

	var int64Buf [8]byte
	binary.BigEndian.PutUint64(int64Buf[:], uint64(ent.Timestamp))

	buf = append(buf, int64Buf[:]...)
	buf = append(buf, byte(ent.Severity))
	buf = writeUtf8(buf, ent.Source)
	buf = append(buf, byte(ent.Encoding))

	binary.BigEndian.PutUint64(int64Buf[:], uint64(len(ent.Message)))
	buf = append(buf, int64Buf[:]...)

	buf = append(buf, ent.Message...)

	payloadLen := int32(len(buf) - entryPayloadBase)
	binary.BigEndian.PutUint32(buf, uint32(payloadLen))

	return buf, payloadLen
}

func writeHop(file *os.File, currentOffset int64, hopStartOffset int64) bool {
	var int32Buf [4]byte

	binary.BigEndian.PutUint32(int32Buf[:], uint32(currentOffset - hopStartOffset))

	if _, err := file.WriteAt(int32Buf[:], hopStartOffset + int64(8)); err != nil {
		println("Failed to write a hop:", err)
		return false
	}

	return true
}

func readEntryHeader(file *os.File, offset int64) (header entryHeader, ok bool) {
	var headerBuf [entryHeaderLength]byte

	if n, _ := file.ReadAt(headerBuf[:], offset); n < entryHeaderLength {
		ok = false
		return
	}

	header.payloadLength = int32(binary.BigEndian.Uint32(headerBuf[:]))
	header.prevPayloadLength = int32(binary.BigEndian.Uint32(headerBuf[4:]))
	header.hop = int32(binary.BigEndian.Uint32(headerBuf[8:]))
	header.backHop = int32(binary.BigEndian.Uint32(headerBuf[12:]))
	header.timestamp = int64(binary.BigEndian.Uint64(headerBuf[16:]))

	ok = true
	return
}

func initLogger(file *os.File) (lastTimestampWritten int64, prevPayloadLen int32, nextHopStartOffset int64, hopCounter int, initialized bool) {
	initialized = true
	lastTimestampWritten = 0
	prevPayloadLen = 0
	nextHopStartOffset = 0
	hopCounter = hopLength

	offset := int64(0)
	var fileSize int64

	if stat, err := file.Stat(); err != nil {
		println("Failed to get stat of a log file:", err)
		initialized = false
		return
	} else {
		fileSize = stat.Size()
	}

	for {
		header, ok := readEntryHeader(file, offset)
		nextOffset := offset + header.NextOffset()

		if !ok || nextOffset > fileSize {
			//The file is broken and needs to be fixed.
			//By truncating the file, we remove the last record
			//that was partially written.
			if err := file.Truncate(offset); err != nil {
				println("Failed to truncate a log file:", err)
				initialized = false
				return
			}

			//Setting the cursor to the end of the file.
			if _, err := file.Seek(0, 2); err != nil {
				println("Failed to seek to the end of a log file:", err)
				initialized = false
				return
			}

			return
		} else if nextOffset == fileSize {
			//Setting the cursor to the end of the file.
			if _, err := file.Seek(0, 2); err != nil {
				println("Failed to seek to the end of a log file:", err)
				initialized = false
			}

			return
		}

		if nextOffset <= offset {
			println("Next offset must be greater than the current: nextOffset =", nextOffset, "offset =", offset)
			panic("Next offset must be greater than the current.")
		}

		if header.hop > 0 {
			hopOffset := offset + int64(header.hop)
			hopCounter = hopLength

			if hopOffset <= fileSize {
				offset = hopOffset
				nextHopStartOffset = offset
			} else {
				nextHopStartOffset = offset
				offset = nextOffset
			}

		} else {
			offset = nextOffset
			hopCounter--
		}

		lastTimestampWritten = header.timestamp
		prevPayloadLen = header.payloadLength
	}

	return
}

func (log *LogFile) run(file *os.File) {
	buf := make([]byte, 0, defaultEntryBufferSize)

	stop := false
	lastTimestampWritten, prevPayloadLen, nextHopStartOffset, hopCounter, initialized := initLogger(file)
	currentOffset, _ := file.Seek(0, 1)

	for !stop {
		select {
			case ent := <- log.writeChan:
				if initialized && ent.Timestamp >= lastTimestampWritten {
					backHop := int32(0)

					if hopCounter--; hopCounter == 0 {
						backHop = int32(currentOffset - nextHopStartOffset)
					}

					buf, prevPayloadLen = writeEntry(buf, ent, prevPayloadLen, backHop)

					if _, err := file.Write(buf); err != nil {
						println("Failed to wrtie log entry:", err)
						currentOffset, _ = file.Seek(0, 1)

					} else {
						if hopCounter == 0 {
							if writeHop(file, currentOffset, nextHopStartOffset) {
								nextHopStartOffset = currentOffset
								hopCounter = hopLength
							}
						}

						currentOffset += int64(len(buf))
					}

					lastTimestampWritten = ent.Timestamp
					buf = buf[:0]
				}

			//case q := <- log.readChan:

			case cmd := <- log.closeChan:
				file.Close()
				cmd.ack <- true
				stop = true
		}
	}
}

func (log *LogFile) WriteLog(entry *LogEntry) {
	log.writeChan <- entry
}

func (log *LogFile) ReadLog(from int64, to int64, minSeverity int, maxSeverity int, source string) chan *LogEntry {
	result := make(chan *LogEntry)
	q := &logQuery { from, to, minSeverity, maxSeverity, source, result}

	log.readChan <- q

	return result
}

func (log *LogFile) Close() {
	ack := make(chan bool)
	closeCmd := &logClose { ack }

	log.closeChan <- closeCmd
	<- ack

	close(log.writeChan)
	close(log.readChan)
	close(log.closeChan)
}<commit_msg>Log entry reads.<commit_after>/*
	This file is part of LogHub.

	LogHub is free software: you can redistribute it and/or modify
	it under the terms of the GNU General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	LogHub is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU General Public License for more details.

	You should have received a copy of the GNU General Public License
	along with LogHub.  If not, see <http://www.gnu.org/licenses/>.
*/
package main

import (
	"os"
	"encoding/binary"
)

const defaultEntryBufferSize = 1024
const entryPayloadBase = 16
const timestampLength = 8
const entryHeaderLength = entryPayloadBase + timestampLength
const hopLength = 64

type logQuery struct {
	from        int64
	to     		int64
	minSeverity int
	maxSeverity int
	source      string
	result      chan *LogEntry
}

type logClose struct {
	ack chan bool
}

type LogFile struct {
	writeChan chan *LogEntry
	readChan  chan *logQuery
	closeChan chan *logClose
}

type entryHeader struct {
	payloadLength     int32
	prevPayloadLength int32
	hop               int32
	backHop           int32
	timestamp         int64
}

func (h *entryHeader) NextOffset() int64 {
	return int64(h.payloadLength) + int64(entryPayloadBase)
}

func (h *entryHeader) PrevOffset() int64 {
	return int64(h.prevPayloadLength) + int64(entryPayloadBase)
}

func OpenCreateLogFile(path string) (*LogFile, error) {
	if f, err := os.OpenFile(path, os.O_RDWR | os.O_CREATE, 0660); err == nil {
		log := &LogFile{ make(chan *LogEntry), make(chan *logQuery), make(chan *logClose) }

		go log.run(f)

		return log, nil
	} else {
		return nil, err
	}
}

func writeUtf8(buf []byte, str string) []byte {
	var int32Buf [4]byte

	binary.BigEndian.PutUint32(int32Buf[:], uint32(len(str)))
	buf = append(buf, int32Buf[:]...)
	buf = append(buf, []byte(str)...)

	return buf
}

func writeEntry(buf []byte, ent *LogEntry, prevPayloadLen int32, backHop int32) ([]byte, int32) {
	for i := 0; i < entryPayloadBase; i++ {
		buf = append(buf, byte(0))
	}

	binary.BigEndian.PutUint32(buf[4:], uint32(prevPayloadLen))
	binary.BigEndian.PutUint32(buf[12:], uint32(backHop))

	var int64Buf [8]byte
	binary.BigEndian.PutUint64(int64Buf[:], uint64(ent.Timestamp))

	buf = append(buf, int64Buf[:]...)
	buf = append(buf, byte(ent.Severity))
	buf = writeUtf8(buf, ent.Source)
	buf = append(buf, byte(ent.Encoding))

	binary.BigEndian.PutUint32(int64Buf[:], uint32(len(ent.Message)))
	buf = append(buf, int64Buf[:4]...)

	buf = append(buf, ent.Message...)

	payloadLen := int32(len(buf) - entryPayloadBase)
	binary.BigEndian.PutUint32(buf, uint32(payloadLen))

	return buf, payloadLen
}

func writeHop(file *os.File, currentOffset int64, hopStartOffset int64) bool {
	var int32Buf [4]byte

	binary.BigEndian.PutUint32(int32Buf[:], uint32(currentOffset - hopStartOffset))

	if _, err := file.WriteAt(int32Buf[:], hopStartOffset + int64(8)); err != nil {
		println("Failed to write a hop:", err)
		return false
	}

	return true
}

func readEntryHeader(file *os.File, offset int64) (header entryHeader, ok bool) {
	var headerBuf [entryHeaderLength]byte

	if n, err := file.ReadAt(headerBuf[:], offset); n < entryHeaderLength {
		println("Failed to read entry header:", err)
		ok = false
		return
	}

	header.payloadLength = int32(binary.BigEndian.Uint32(headerBuf[:]))
	header.prevPayloadLength = int32(binary.BigEndian.Uint32(headerBuf[4:]))
	header.hop = int32(binary.BigEndian.Uint32(headerBuf[8:]))
	header.backHop = int32(binary.BigEndian.Uint32(headerBuf[12:]))
	header.timestamp = int64(binary.BigEndian.Uint64(headerBuf[16:]))

	ok = true
	return
}

func readBytes(file *os.File, offset int64) ([]byte, error) {
	var int32Buf [4]byte

	if _, err := file.ReadAt(int32Buf[:], offset); err != nil {
		return nil, err
	}

	l := binary.BigEndian.Uint32(int32Buf[:])
	result := make([]byte, l)

	if _, err := file.ReadAt(result, offset + int64(4)); err != nil {
		return nil, err
	}

	return result, nil
}

func readEntry(file *os.File, offset int64) (ent *LogEntry, ok bool) {
	var int64Buf [8]byte

	ent = new(LogEntry)
	ok = true

	offset += int64(entryPayloadBase)

	if _, err := file.ReadAt(int64Buf[:], offset); err != nil {
		println("Failed to read log entry:", err)
		ok = false
		return
	}

	ent.Timestamp = int64(binary.BigEndian.Uint64(int64Buf[:]))
	offset += 8

	if _, err := file.ReadAt(int64Buf[:1], offset); err != nil {
		println("Failed to read log entry:", err)
		ok = false
		return
	}

	ent.Severity = int(int64Buf[0])
	offset += 1

	if s, err := readBytes(file, offset); err != nil {
		println("Failed to read log entry:", err)
		ok = false
		return
	} else {
		ent.Source = string(s)
		offset += int64(len(s))
	}

	if _, err := file.ReadAt(int64Buf[:1], offset); err != nil {
		println("Failed to read log entry:", err)
		ok = false
		return
	}

	ent.Encoding = int(int64Buf[0])
	offset += 1

	if m, err := readBytes(file, offset); err != nil {
		println("Failed to read log entry:", err)
		ok = false
		return
	} else {
		ent.Message = m
	}

	return
}

func findLeftBoundFromLeft(file *os.File, q *logQuery, offset int64, lastEntryOffset int64) {

	//Go right without hops.
	for {
		header, ok := readEntryHeader(file, offset)

		if !ok {
			close(q.result)
			return
		}

		if header.timestamp >= q.from && header.timestamp <= q.to {
			readEntries(file, q, offset, lastEntryOffset)
			return
		}

		if offset == lastEntryOffset {
			close(q.result)
			return
		}

		offset += header.NextOffset()
	}
}

func findLeftBoundFromRight(file *os.File, q *logQuery, offset int64, lastEntryOffset int64) {
	prevOffset := offset

	//Go left without hops.
	for {
		header, ok := readEntryHeader(file, offset)

		if !ok {
			close(q.result)
			return
		}

		if header.timestamp < q.from {
			readEntries(file, q, prevOffset, lastEntryOffset)
			return
		} else if offset == 0 {
			readEntries(file, q, offset, lastEntryOffset)
			return
		}

		prevOffset = offset
		offset -= header.PrevOffset()
	}
}

func readEntries(file *os.File, q *logQuery, offset int64, lastEntryOffset int64) {
	for {
		header, ok := readEntryHeader(file, offset)

		if !ok {
			close(q.result)
			return
		}

		if header.timestamp >= q.from && header.timestamp <= q.to {
			if ent, ok := readEntry(file, offset); ok {
				//TODO: Filtering.

				q.result <- ent
			} else {
				close(q.result)
				return
			}
		} else {
			close(q.result)
			return
		}

		if offset < lastEntryOffset {
			offset += header.NextOffset()
		} else {
			close(q.result)
			return
		}
	}
}

func findAndReadEntries(file *os.File, lastEntryOffset int64, q *logQuery) {
	offset := lastEntryOffset

	//Go left doing hops.
	for {
		header, ok := readEntryHeader(file, offset)

		if !ok {
			close(q.result)
			return
		}

		if header.timestamp < q.from {
			findLeftBoundFromLeft(file, q, offset, lastEntryOffset)
			return
		} else if header.timestamp >= q.from && header.timestamp <= q.to {
			findLeftBoundFromRight(file, q, offset, lastEntryOffset)
			return
		}

		if offset == 0 {
			close(q.result)
			return
		}

		if header.backHop != 0 {
			offset -= int64(header.backHop)
		} else {
			offset -= header.PrevOffset()
		}
	}
}

func initLogger(file *os.File) (lastTimestampWritten int64, prevPayloadLen int32, nextHopStartOffset int64, hopCounter int, initialized bool) {
	initialized = true
	lastTimestampWritten = 0
	prevPayloadLen = 0
	nextHopStartOffset = 0
	hopCounter = hopLength

	offset := int64(0)
	var fileSize int64

	if stat, err := file.Stat(); err != nil {
		println("Failed to get stat of a log file:", err)
		initialized = false
		return
	} else {
		fileSize = stat.Size()
	}

	for {
		header, ok := readEntryHeader(file, offset)
		nextOffset := offset + header.NextOffset()

		if !ok || nextOffset > fileSize {
			//The file is broken and needs to be fixed.
			//By truncating the file, we remove the last record
			//that was partially written.
			if err := file.Truncate(offset); err != nil {
				println("Failed to truncate a log file:", err)
				initialized = false
				return
			}

			//Setting the cursor to the end of the file.
			if _, err := file.Seek(0, 2); err != nil {
				println("Failed to seek to the end of a log file:", err)
				initialized = false
				return
			}

			return
		} else if nextOffset == fileSize {
			//Setting the cursor to the end of the file.
			if _, err := file.Seek(0, 2); err != nil {
				println("Failed to seek to the end of a log file:", err)
				initialized = false
			}

			return
		}

		if nextOffset <= offset {
			println("Next offset must be greater than the current: nextOffset =", nextOffset, "offset =", offset)
			panic("Next offset must be greater than the current.")
		}

		if header.hop > 0 {
			hopOffset := offset + int64(header.hop)
			hopCounter = hopLength

			if hopOffset <= fileSize {
				offset = hopOffset
				nextHopStartOffset = offset
			} else {
				nextHopStartOffset = offset
				offset = nextOffset
			}

		} else {
			offset = nextOffset
			hopCounter--
		}

		lastTimestampWritten = header.timestamp
		prevPayloadLen = header.payloadLength
	}

	return
}

func (log *LogFile) run(file *os.File) {
	buf := make([]byte, 0, defaultEntryBufferSize)

	stop := false
	lastTimestampWritten, prevPayloadLen, nextHopStartOffset, hopCounter, initialized := initLogger(file)
	currentOffset, _ := file.Seek(0, 1)

	for !stop {
		select {
			case ent := <- log.writeChan:
				if initialized && ent.Timestamp >= lastTimestampWritten {
					backHop := int32(0)

					if hopCounter--; hopCounter == 0 {
						backHop = int32(currentOffset - nextHopStartOffset)
					}

					buf, prevPayloadLen = writeEntry(buf, ent, prevPayloadLen, backHop)

					if _, err := file.Write(buf); err != nil {
						println("Failed to wrtie log entry:", err)
						currentOffset, _ = file.Seek(0, 1)

					} else {
						if hopCounter == 0 {
							if writeHop(file, currentOffset, nextHopStartOffset) {
								nextHopStartOffset = currentOffset
								hopCounter = hopLength
							}
						}

						currentOffset += int64(len(buf))
					}

					lastTimestampWritten = ent.Timestamp
					buf = buf[:0]
				}

			case q := <- log.readChan:
				//No entry will pass the filter.
				if q.from >= q.to || q.minSeverity > q.maxSeverity {
					close(q.result)
				}

				lastEntryOffset := currentOffset - int64(prevPayloadLen - entryPayloadBase)

				//TODO: Sync. with closing.
				go findAndReadEntries(file, lastEntryOffset, q)

			case cmd := <- log.closeChan:
				//TODO: Wait reads.

				file.Close()
				cmd.ack <- true
				stop = true
		}
	}
}

func (log *LogFile) WriteLog(entry *LogEntry) {
	log.writeChan <- entry
}

func (log *LogFile) ReadLog(from int64, to int64, minSeverity int, maxSeverity int, source string) chan *LogEntry {
	result := make(chan *LogEntry)
	q := &logQuery { from, to, minSeverity, maxSeverity, source, result}

	log.readChan <- q

	return result
}

func (log *LogFile) Close() {
	ack := make(chan bool)
	closeCmd := &logClose { ack }

	log.closeChan <- closeCmd
	<- ack

	close(log.writeChan)
	close(log.readChan)
	close(log.closeChan)
}<|endoftext|>