<commit_before>package onedriveclient

import (
	"encoding/json"
	"fmt"
	"github.com/koofr/go-httpclient"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"path"
	"strings"
	"time"
)

type OneDrive struct {
	ApiClient     *httpclient.HTTPClient
	ContentClient *httpclient.HTTPClient
	Auth          *OneDriveAuth
}

type OneDriveAuth struct {
	ClientId     string
	ClientSecret string
	RedirectUri  string
	AccessToken  string
	RefreshToken string
	ExpiresAt    time.Time
}

func (d *OneDriveAuth) validToken() (token string, err error) {
	if time.Now().Unix() > d.ExpiresAt.Unix() {
		var resp *http.Response
		resp, err = http.PostForm("https://login.live.com/oauth20_token.srf",
			url.Values{
				"grant_type":    {"refresh_token"},
				"client_id":     {d.ClientId},
				"client_secret": {d.ClientSecret},
				"redirect_uri":  {d.RedirectUri},
				"refresh_token": {d.RefreshToken},
			})
		if err != nil {
			return
		}

		if resp.StatusCode != 200 {
			err = fmt.Errorf("Token refresh failed %d: %s", resp.StatusCode, resp.Status)
			return
		}

		var buf []byte
		if buf, err = ioutil.ReadAll(resp.Body); err != nil {
			return
		}

		var respVal RefreshResp
		if err = json.Unmarshal(buf, &respVal); err != nil {
			return
		}

		d.AccessToken = respVal.AccessToken
		d.ExpiresAt = time.Now().Add(time.Duration(respVal.ExpiresIn) * time.Second)
	}
	token = d.AccessToken
	return
}

func NewOneDriveClient(auth OneDriveAuth) *OneDrive {
	apiBaseUrl, _ := url.Parse("https://apis.live.net/v5.0")
	apiHttpClient := httpclient.New()
	apiHttpClient.BaseURL = apiBaseUrl
	return &OneDrive{apiHttpClient, httpclient.New(), &auth}
}

func (d *OneDrive) authenticationHeader() (hs http.Header, err error) {
	token, err := d.Auth.validToken()
	if err != nil {
		return
	}

	hs = make(http.Header)
	hs.Set("Authorization", "Bearer "+token)
	return
}

func (d *OneDrive) NodeInfo(id string) (info NodeInfo, err error) {
	header, err := d.authenticationHeader()
	if err != nil {
		return
	}

	req := &httpclient.RequestData{
		Method:         "GET",
		Path:           "/" + id,
		Headers:        header,
		ExpectedStatus: []int{200},
		RespEncoding:   httpclient.EncodingJSON,
		RespValue:      &info,
	}
	_, err = d.ApiClient.Request(req)
	if err != nil {
		return
	}
	return
}

func (d *OneDrive) RootInfo() (info NodeInfo, err error) {
	info, err = d.NodeInfo("me/skydrive")
	return
}

func (d *OneDrive) NodeFiles(id string) (files []NodeInfo, err error) {
	header, err := d.authenticationHeader()
	if err != nil {
		return
	}

	var resp NodeFiles
	req := &httpclient.RequestData{
		Method:         "GET",
		Path:           "/" + id + "/files",
		Headers:        header,
		ExpectedStatus: []int{200},
		RespEncoding:   httpclient.EncodingJSON,
		RespValue:      &resp,
	}
	_, err = d.ApiClient.Request(req)
	if err != nil {
		return
	}

	files = resp.Data
	return
}

func (d *OneDrive) Download(id string) (info NodeInfo, content io.ReadCloser, err error) {
	info, err = d.NodeInfo(id)
	if err != nil {
		return
	}

	url := info.Source
	if url == "" {
		err = fmt.Errorf("Cannot download %s", id)
		return
	}

	req := httpclient.RequestData{
		Method:         "GET",
		FullURL:        url,
		ExpectedStatus: []int{http.StatusOK},
	}

	res, err := d.ContentClient.Request(&req)
	if err != nil {
		return
	}

	content = res.Body
	return
}

func (d *OneDrive) ResolvePath(pth string) (id string, err error) {
	root, err := d.RootInfo()
	if err != nil {
		return
	}
	id = root.Id

loopParts:
	for _, part := range pathParts(pth) {
		var files []NodeInfo
		files, err = d.NodeFiles(id)
		if err != nil {
			return
		}
		name := strings.ToLower(part)
		for _, file := range files {
			if strings.ToLower(file.Name) == name {
				id = file.Id
				continue loopParts
			}
		}
		return "", fmt.Errorf("Not found %s in %s", part, files)
	}
	return
}

func pathParts(pth string) []string {
	pth = path.Clean("/" + pth)
	parts := make([]string, 0)
	for pth != "/" {
		var name string
		pth, name = path.Split(pth)
		pth = path.Clean(pth)
		parts = append(parts, name)
	}

	//in-place reverse
	l := len(parts) - 1
	h := len(parts) / 2
	for i := 0; i < h; i++ {
		t := parts[i]
		ii := l - i
		parts[i] = parts[ii]
		parts[ii] = t
	}
	return parts
}
<commit_msg>upload capabilities<commit_after>package onedriveclient

import (
	"encoding/json"
	"fmt"
	"github.com/koofr/go-httpclient"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"path"
	"strings"
	"time"
)

type OneDrive struct {
	ApiClient     *httpclient.HTTPClient
	ContentClient *httpclient.HTTPClient
	Auth          *OneDriveAuth
}

type OneDriveAuth struct {
	ClientId     string
	ClientSecret string
	RedirectUri  string
	AccessToken  string
	RefreshToken string
	ExpiresAt    time.Time
}

func (d *OneDriveAuth) validToken() (token string, err error) {
	if time.Now().Unix() > d.ExpiresAt.Unix() {
		var resp *http.Response
		resp, err = http.PostForm("https://login.live.com/oauth20_token.srf",
			url.Values{
				"grant_type":    {"refresh_token"},
				"client_id":     {d.ClientId},
				"client_secret": {d.ClientSecret},
				"redirect_uri":  {d.RedirectUri},
				"refresh_token": {d.RefreshToken},
			})
		if err != nil {
			return
		}

		if resp.StatusCode != 200 {
			err = fmt.Errorf("Token refresh failed %d: %s", resp.StatusCode, resp.Status)
			return
		}

		var buf []byte
		if buf, err = ioutil.ReadAll(resp.Body); err != nil {
			return
		}

		var respVal RefreshResp
		if err = json.Unmarshal(buf, &respVal); err != nil {
			return
		}

		d.AccessToken = respVal.AccessToken
		d.ExpiresAt = time.Now().Add(time.Duration(respVal.ExpiresIn) * time.Second)
	}
	token = d.AccessToken
	return
}

func NewOneDriveClient(auth OneDriveAuth) *OneDrive {
	apiBaseUrl, _ := url.Parse("https://apis.live.net/v5.0")
	apiHttpClient := httpclient.New()
	apiHttpClient.BaseURL = apiBaseUrl
	return &OneDrive{apiHttpClient, httpclient.New(), &auth}
}

func (d *OneDrive) authenticationHeader() (hs http.Header, err error) {
	token, err := d.Auth.validToken()
	if err != nil {
		return
	}

	hs = make(http.Header)
	hs.Set("Authorization", "Bearer "+token)
	return
}

func (d *OneDrive) NodeInfo(id string) (info NodeInfo, err error) {
	header, err := d.authenticationHeader()
	if err != nil {
		return
	}

	req := &httpclient.RequestData{
		Method:         "GET",
		Path:           "/" + id,
		Headers:        header,
		ExpectedStatus: []int{200},
		RespEncoding:   httpclient.EncodingJSON,
		RespValue:      &info,
	}
	_, err = d.ApiClient.Request(req)
	if err != nil {
		return
	}
	return
}

func (d *OneDrive) RootInfo() (info NodeInfo, err error) {
	info, err = d.NodeInfo("me/skydrive")
	return
}

func (d *OneDrive) NodeFiles(id string) (files []NodeInfo, err error) {
	header, err := d.authenticationHeader()
	if err != nil {
		return
	}

	var resp NodeFiles
	req := &httpclient.RequestData{
		Method:         "GET",
		Path:           "/" + id + "/files",
		Headers:        header,
		ExpectedStatus: []int{200},
		RespEncoding:   httpclient.EncodingJSON,
		RespValue:      &resp,
	}
	_, err = d.ApiClient.Request(req)
	if err != nil {
		return
	}

	files = resp.Data
	return
}

func (d *OneDrive) Download(id string) (info NodeInfo, content io.ReadCloser, err error) {
	info, err = d.NodeInfo(id)
	if err != nil {
		return
	}

	url := info.Source
	if url == "" {
		err = fmt.Errorf("Cannot download %s", id)
		return
	}

	req := httpclient.RequestData{
		Method:         "GET",
		FullURL:        url,
		ExpectedStatus: []int{http.StatusOK},
	}

	res, err := d.ContentClient.Request(&req)
	if err != nil {
		return
	}

	content = res.Body
	return
}

func (d *OneDrive) Upload(dirId string, name string, content io.Reader) (err error) {
	header, err := d.authenticationHeader()
	if err != nil {
		return
	}

	req := httpclient.RequestData{
		Method:         "PUT",
		Path:           "/" + dirId + "/files/" + name,
		Headers:        header,
		ReqReader:      content,
		ExpectedStatus: []int{200, 201},
	}

	_, err = d.ApiClient.Request(&req)
	if err != nil {
		return
	}

	return
}

func (d *OneDrive) ResolvePath(pth string) (id string, err error) {
	root, err := d.RootInfo()
	if err != nil {
		return
	}
	id = root.Id

loopParts:
	for _, part := range pathParts(pth) {
		var files []NodeInfo
		files, err = d.NodeFiles(id)
		if err != nil {
			return
		}
		name := strings.ToLower(part)
		for _, file := range files {
			if strings.ToLower(file.Name) == name {
				id = file.Id
				continue loopParts
			}
		}
		return "", fmt.Errorf("Not found %s in %s", part, files)
	}
	return
}

func pathParts(pth string) []string {
	pth = path.Clean("/" + pth)
	parts := make([]string, 0)
	for pth != "/" {
		var name string
		pth, name = path.Split(pth)
		pth = path.Clean(pth)
		parts = append(parts, name)
	}

	//in-place reverse
	l := len(parts) - 1
	h := len(parts) / 2
	for i := 0; i < h; i++ {
		t := parts[i]
		ii := l - i
		parts[i] = parts[ii]
		parts[ii] = t
	}
	return parts
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package test

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"

	"github.com/GoogleCloudPlatform/terraform-validator/converters/google"
)

// TestConvert tests the "convert" subcommand against a generated .tfplan file.
func TestConvert(t *testing.T) {
	_, cfg := setup(t)

	cmd := exec.Command(filepath.Join("..", "bin", "terraform-validator"),
		"convert",
		"--project", cfg.project,
		planPath,
	)
	cmd.Env = []string{"GOOGLE_APPLICATION_CREDENTIALS=" + cfg.credentials}
	var stderr, stdout bytes.Buffer
	cmd.Stderr, cmd.Stdout = &stderr, &stdout

	if err := cmd.Run(); err != nil {
		t.Fatalf("%v:\n%v", err, stderr.String())
	}

	var assets []google.Asset
	if err := json.Unmarshal(stdout.Bytes(), &assets); err != nil {
		t.Fatalf("unmarshaling: %v", err)
	}

	assetsByType := make(map[string][]google.Asset)
	for _, a := range assets {
		assetsByType[a.Type] = append(assetsByType[a.Type], a)
	}

	jsonFixtures := make(map[string][]byte)

	matches, _ := filepath.Glob(filepath.Join(jsonGenerateDir, "*.json"))
	for _, fixturePath := range matches {
		fixtureFileName := strings.TrimPrefix(fixturePath, jsonGenerateDir+"/")
		fixtureName := strings.TrimSuffix(fixtureFileName, ".json")

		fixtureData, err := ioutil.ReadFile(fixturePath)
		if err != nil {
			t.Fatalf("Error reading %v: %v", fixturePath, err)
		}

		jsonFixtures[fixtureName] = fixtureData
	}

	t.Run("Disk", func(t *testing.T) {
		requireEqualJSON(t,
			jsonFixtures["disk"],
			assetsByType["compute.googleapis.com/Disk"][0].Resource.Data,
		)
	})

	t.Run("Project", func(t *testing.T) {
		requireEqualJSON(t,
			jsonFixtures["project"],
			assetsByType["cloudresourcemanager.googleapis.com/Project"][0].Resource.Data,
		)
	})

	t.Run("ProjectBillingInfo", func(t *testing.T) {
		requireEqualJSON(t,
			jsonFixtures["project_billing_info"],
			assetsByType["cloudbilling.googleapis.com/ProjectBillingInfo"][0].Resource.Data,
		)
	})

	t.Run("Firewall", func(t *testing.T) {
		requireEqualJSON(t,
			jsonFixtures["firewall"],
			assetsByType["compute.googleapis.com/Firewall"][0].Resource.Data,
		)
	})
}
<commit_msg>Convert integration tests to table-driven tests<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package test

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"

	"github.com/GoogleCloudPlatform/terraform-validator/converters/google"
)

var conversionTests = []struct {
	name      string
	assetType string
}{
	{"disk", "compute.googleapis.com/Disk"},
	{"project", "cloudresourcemanager.googleapis.com/Project"},
	{"project_billing_info", "cloudbilling.googleapis.com/ProjectBillingInfo"},
	{"firewall", "compute.googleapis.com/Firewall"},
}

// TestConvert tests the "convert" subcommand against a generated .tfplan file.
func TestConvert(t *testing.T) {
	_, cfg := setup(t)

	cmd := exec.Command(filepath.Join("..", "bin", "terraform-validator"),
		"convert",
		"--project", cfg.project,
		planPath,
	)
	cmd.Env = []string{"GOOGLE_APPLICATION_CREDENTIALS=" + cfg.credentials}
	var stderr, stdout bytes.Buffer
	cmd.Stderr, cmd.Stdout = &stderr, &stdout

	if err := cmd.Run(); err != nil {
		t.Fatalf("%v:\n%v", err, stderr.String())
	}

	var assets []google.Asset
	if err := json.Unmarshal(stdout.Bytes(), &assets); err != nil {
		t.Fatalf("unmarshaling: %v", err)
	}

	assetsByType := make(map[string][]google.Asset)
	for _, a := range assets {
		assetsByType[a.Type] = append(assetsByType[a.Type], a)
	}

	jsonFixtures := make(map[string][]byte)

	matches, _ := filepath.Glob(filepath.Join(jsonGenerateDir, "*.json"))
	for _, fixturePath := range matches {
		fixtureFileName := strings.TrimPrefix(fixturePath, jsonGenerateDir+"/")
		fixtureName := strings.TrimSuffix(fixtureFileName, ".json")

		fixtureData, err := ioutil.ReadFile(fixturePath)
		if err != nil {
			t.Fatalf("Error reading %v: %v", fixturePath, err)
		}

		jsonFixtures[fixtureName] = fixtureData
	}

	for _, tt := range conversionTests {
		// actual := assetsByType[tt.assetType][0].Resource.Data
		t.Run(tt.name, func(t *testing.T) {
			requireEqualJSON(t,
				jsonFixtures[tt.name],
				assetsByType[tt.assetType][0].Resource.Data,
			)
		})
	}
}
<|endoftext|>


<commit_before>/*
This package allows for easy validation of passed in json.
Val does not intend to be a robust solution but does seek to cover 95% of use cases.
Val requires a structure to use pointers for validation. This may seem odd but if a pointer is
not used you will run into some strange issues since json.Decode() will pass an int type back
set as 0 giving no way to tell if a 0 was actually passed in or not. Using a pointer allows to
check for a nil value before doing the validation and lets you have optional json parameters.

Basic Struct Example.

    var Register struct {
        Username *string `json:"username" validate:"required"`
        Password *string `json:"password" validate:"required"`
        Email    *string `json:"email" validate:"required|email"`
        Notify   *string `json:"notify" validate:"required|in:yes,no"`
    }

Normal Use Case.

    if err := val.Bind(r.Body, &Register); err != nil {
        fmt.Println(err)
    }

*/

package val

import (
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"reflect"
	"regexp"
	"strconv"
	"strings"
)

// Unpack JSON and call the validate function if no errors are found when unpacking it.
// Bind kicks of the validation process. Note that Request.Body impliments an io.ReadCloser.
// Look into ReadAll http://jmoiron.net/blog/crossing-streams-a-love-letter-to-ioreader/
func Bind(input io.ReadCloser, obj interface{}) error {
	// Don't go through any logic if nothing was passed in.
	if b, err := ioutil.ReadAll(input); err == nil && string(b) != "{}" && string(b) != "" {
		// Turn our string back into a io.Reader if it's valid
		decoder := json.NewDecoder(bytes.NewReader(b))

		if err := decoder.Decode(obj); err == nil {
			return Validate(obj)
		} else {
			return err
		}
	} else if err == nil {
		return errors.New("Nothing was passed in or JSON featured an empty object.")
	} else {
		return err
	}
}

// In version 1.0 I exported the Validation function. This can be used when you may
// not need to or want to have JSON first converted into a struct.
func Validate(obj interface{}) error {

	typ := reflect.TypeOf(obj)
	value := reflect.ValueOf(obj)

	// Check to ensure we are getting a valid
	// pointer for manipulation.
	if typ.Kind() == reflect.Ptr {
		typ = typ.Elem()
		value = value.Elem()
	}

	// Kill process if obj did not pass in a scruct.
	// This happens when a pointer passed in.
	if value.Kind() != reflect.Struct {
		return nil
	}

	for i := 0; i < typ.NumField(); i++ {

		field := typ.Field(i)
		fieldValue := value.Field(i).Interface()
		zero := reflect.Zero(field.Type).Interface()

		// Validate nested and embedded structs (if pointer, only do so if not nil)
		if field.Type.Kind() == reflect.Struct ||
			(field.Type.Kind() == reflect.Ptr && !reflect.DeepEqual(zero, fieldValue)) {
			if err := Validate(fieldValue); err != nil {
				return err
			}
		}

		if field.Tag.Get("validate") != "" || field.Tag.Get("binding") != "" {
			// Break validate field into array
			array := strings.Split(field.Tag.Get("validate"), "|")

			// Legacy Support for binding.
			if array[0] == "" {
				array = strings.Split(field.Tag.Get("binding"), "|")
			}

			// Do the hard work of checking all assertions
			for setting := range array {

				match := array[setting]

				//Check that value was passed in and is not required.
				if match != "required" && null(fieldValue) == true {
					return nil
				}

				switch {
				case "required" == match:
					if err := required(field, fieldValue, zero); err != nil {
						return err
					}
				case "email" == match:
					if err := regex(`regex:^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$`, fieldValue); err != nil {
						return err
					}
				case strings.HasPrefix(match, "min:"):
					if err := min(match, fieldValue); err != nil {
						return err
					}
				case strings.HasPrefix(match, "max:"):
					if err := max(match, fieldValue); err != nil {
						return err
					}
				case strings.HasPrefix(match, "in:"):
					if err := in(match, fieldValue); err != nil {
						return err
					}
				case strings.HasPrefix(match, "regex:"):
					if err := regex(match, fieldValue); err != nil {
						return err
					}
				case strings.HasPrefix(match, "length:"):
					if err := length(match, fieldValue); err != nil {
						return err
					}
				case strings.HasPrefix(match, "length_between:"):
					if err := length_between(match, fieldValue); err != nil {
						return err
					}
				default:
					panic("The field " + match + " is not a valid validation check.")
				}
			}
		}
	}

	return nil
}

// Ensure that the value being passed in is not of type nil.
func null(value interface{}) bool {
	if reflect.ValueOf(value).IsNil() {
		return true
	}

	return false
}

// Check that the following function features
// the required field. May need to check for
// more special cases like since passing in null
// is the same as 0 for int type checking.
func required(field reflect.StructField, value, zero interface{}) error {

	if reflect.DeepEqual(zero, value) {
		if _, ok := value.(int); !ok {
			return errors.New("The required field " + field.Name + " was not submitted.")
		}
	}

	return nil
}

// Check that the passed in field is a valid email
// Need to improve error logging for this method
// Currently only supports strings, ints
func in(field string, value interface{}) error {

	if data, ok := value.(*string); ok {
		if len(*data) == 0 {
			return nil
		}

		valid := strings.Split(field[3:], ",")

		for option := range valid {
			if valid[option] == *data {
				return nil
			}
		}

	} else {
		return errors.New("The value passed in for IN could not be converted to a string.")
	}

	return errors.New("In did not match any of the expected values.")
}

func min(field string, value interface{}) error {

	if data, ok := value.(*int); ok {

		min := field[strings.Index(field, ":")+1:]

		if minNum, ok := strconv.ParseInt(min, 0, 64); ok == nil {

			if int64(*data) >= minNum {
				return nil
			} else {
				return errors.New("The data you passed in was smaller then the allowed minimum.")
			}

		}
	}

	return errors.New("The value passed in for MIN could not be converted to an int.")
}

func max(field string, value interface{}) error {

	if data, ok := value.(*int); ok {

		max := field[strings.Index(field, ":")+1:]

		if maxNum, ok := strconv.ParseInt(max, 0, 64); ok == nil {
			if int64(*data) <= maxNum {
				return nil
			} else {
				return errors.New("The data you passed in was larger than the maximum.")
			}

		}
	}

	return errors.New("The value passed in for MAX could not be converted to an int.")
}

// Regex handles the general regex call and also handles
// the regex email.
func regex(field string, value interface{}) error {

	reg := field[strings.Index(field, ":")+1:]

	if data, ok := value.(*string); ok {
		if len(*data) == 0 {
			return nil
		} else if err := match_regex(reg, []byte(*data)); err != nil {
			return err
		}
	} else if data, ok := value.(*int); ok {
		if err := match_regex(reg, []byte(strconv.Itoa(*data))); err != nil {
			return err
		}
	} else {
		return errors.New("The value passed in for REGEX could not be converted to a string or int.")
	}

	return nil
}

// Helper function for regex.
func match_regex(reg string, data []byte) error {

	if match, err := regexp.Match(reg, []byte(data)); err == nil && match {
		return nil
	} else {
		return errors.New("Your regex did not match or was not valid.")
	}
}

// Check passed in json length string is exact value passed in.
// Also checks if passed in values is between two different ones.
func length(field string, value interface{}) error {

	length := field[strings.Index(field, ":")+1:]

	if data, ok := value.(*string); ok {
		if intdata, intok := strconv.Atoi(length); intok == nil {
			if len(*data) == intdata {
				return nil
			} else {
				return errors.New("The data passed in was not equal to the expected length.")
			}
		} else {
			return errors.New("The value passed in for LENGTH could not be converted to an int.")
		}
	} else {
		return errors.New("The value passed in for LENGTH could not be converted to a string.")
	}
}

// Check if the strings length is between high,low.
func length_between(field string, value interface{}) error {

	length := field[strings.Index(field, ":")+1:]
	vals := strings.Split(length, ",")

	if len(vals) == 2 {

		if data, ok := value.(*string); ok {

			if lowerbound, lowok := strconv.Atoi(vals[0]); lowok == nil {

				if upperbound, upok := strconv.Atoi(vals[1]); upok == nil {

					if lowerbound <= len(*data) && upperbound >= len(*data) {
						return nil
					} else {
						return errors.New("The value passed in for LENGTH BETWEEN was not in bounds.")
					}

				} else {
					return errors.New("The value passed in for LENGTH BETWEEN could not be converted to an int.")
				}

			} else {
				return errors.New("The value passed in for LENGTH BETWEEN could not be converted to an int.")
			}

		} else {
			return errors.New("The value passed in for LENGTH BETWEEN could not be converted to a string.")
		}
	} else {
		return errors.New("LENGTH BETWEEN requires exactly two paramaters.")
	}
}
<commit_msg>added in alpha, alphadash, alphanumeric, url support<commit_after>/*
This package allows for easy validation of passed in json.
Val does not intend to be a robust solution but does seek to cover 95% of use cases.
Val requires a structure to use pointers for validation. This may seem odd but if a pointer is
not used you will run into some strange issues since json.Decode() will pass an int type back
set as 0 giving no way to tell if a 0 was actually passed in or not. Using a pointer allows to
check for a nil value before doing the validation and lets you have optional json parameters.

Basic Struct Example.

    var Register struct {
        Username *string `json:"username" validate:"required"`
        Password *string `json:"password" validate:"required"`
        Email    *string `json:"email" validate:"required|email"`
        Notify   *string `json:"notify" validate:"required|in:yes,no"`
    }

Normal Use Case.

    if err := val.Bind(r.Body, &Register); err != nil {
        fmt.Println(err)
    }

*/

package val

import (
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"reflect"
	"regexp"
	"strconv"
	"strings"
)

// Unpack JSON and call the validate function if no errors are found when unpacking it.
// Bind kicks of the validation process. Note that Request.Body impliments an io.ReadCloser.
// Look into ReadAll http://jmoiron.net/blog/crossing-streams-a-love-letter-to-ioreader/
func Bind(input io.ReadCloser, obj interface{}) error {
	// Don't go through any logic if nothing was passed in.
	if b, err := ioutil.ReadAll(input); err == nil && string(b) != "{}" && string(b) != "" {
		// Turn our string back into a io.Reader if it's valid
		decoder := json.NewDecoder(bytes.NewReader(b))

		if err := decoder.Decode(obj); err == nil {
			return Validate(obj)
		} else {
			return err
		}
	} else if err == nil {
		return errors.New("Nothing was passed in or JSON featured an empty object.")
	} else {
		return err
	}
}

// In version 1.0 I exported the Validation function. This can be used when you may
// not need to or want to have JSON first converted into a struct.
func Validate(obj interface{}) error {

	typ := reflect.TypeOf(obj)
	value := reflect.ValueOf(obj)

	// Check to ensure we are getting a valid
	// pointer for manipulation.
	if typ.Kind() == reflect.Ptr {
		typ = typ.Elem()
		value = value.Elem()
	}

	// Kill process if obj did not pass in a scruct.
	// This happens when a pointer passed in.
	if value.Kind() != reflect.Struct {
		return nil
	}

	for i := 0; i < typ.NumField(); i++ {

		field := typ.Field(i)
		fieldValue := value.Field(i).Interface()
		zero := reflect.Zero(field.Type).Interface()

		// Validate nested and embedded structs (if pointer, only do so if not nil)
		if field.Type.Kind() == reflect.Struct ||
			(field.Type.Kind() == reflect.Ptr && !reflect.DeepEqual(zero, fieldValue)) {
			if err := Validate(fieldValue); err != nil {
				return err
			}
		}

		if field.Tag.Get("validate") != "" || field.Tag.Get("binding") != "" {
			// Break validate field into array
			array := strings.Split(field.Tag.Get("validate"), "|")

			// Legacy Support for binding.
			if array[0] == "" {
				array = strings.Split(field.Tag.Get("binding"), "|")
			}

			// Do the hard work of checking all assertions
			for setting := range array {

				match := array[setting]

				//Check that value was passed in and is not required.
				if match != "required" && null(fieldValue) == true {
					return nil
				}

				switch {
				case "required" == match:
					if err := required(field, fieldValue, zero); err != nil {
						return err
					}
				case "email" == match:
					if err := regex(`regex:^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$`, fieldValue); err != nil {
						return err
					}
				case "url" == match:
					if err := regex(`regex:/^(https?:\/\/)?([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w \.-]*)*\/?$/`, fieldValue); err != nil {
						return err
					}
				case "alpha" == match:
					if err := regex(`regex:\p{L}`, fieldValue); err != nil {
						return err
					}
				case "alphadash" == match:
					if err := regex(`regex:^[a-zA-Z0-9_]*$`, fieldValue); err != nil {
						return err
					}
				case "alphanumeric" == match:
					if err := regex(`regex:/[0-9a-zA-Z]/`, fieldValue); err != nil {
						return err
					}
				case strings.HasPrefix(match, "min:"):
					if err := min(match, fieldValue); err != nil {
						return err
					}
				case strings.HasPrefix(match, "max:"):
					if err := max(match, fieldValue); err != nil {
						return err
					}
				case strings.HasPrefix(match, "in:"):
					if err := in(match, fieldValue); err != nil {
						return err
					}
				case strings.HasPrefix(match, "regex:"):
					if err := regex(match, fieldValue); err != nil {
						return err
					}
				case strings.HasPrefix(match, "length:"):
					if err := length(match, fieldValue); err != nil {
						return err
					}
				case strings.HasPrefix(match, "length_between:"):
					if err := length_between(match, fieldValue); err != nil {
						return err
					}
				default:
					panic("The field " + match + " is not a valid validation check.")
				}
			}
		}
	}

	return nil
}

// Ensure that the value being passed in is not of type nil.
func null(value interface{}) bool {
	if reflect.ValueOf(value).IsNil() {
		return true
	}

	return false
}

// Check that the following function features
// the required field. May need to check for
// more special cases like since passing in null
// is the same as 0 for int type checking.
func required(field reflect.StructField, value, zero interface{}) error {

	if reflect.DeepEqual(zero, value) {
		if _, ok := value.(int); !ok {
			return errors.New("The required field " + field.Name + " was not submitted.")
		}
	}

	return nil
}

// Check that the passed in field is a valid email
// Need to improve error logging for this method
// Currently only supports strings, ints
func in(field string, value interface{}) error {

	if data, ok := value.(*string); ok {
		if len(*data) == 0 {
			return nil
		}

		valid := strings.Split(field[3:], ",")

		for option := range valid {
			if valid[option] == *data {
				return nil
			}
		}

	} else {
		return errors.New("The value passed in for IN could not be converted to a string.")
	}

	return errors.New("In did not match any of the expected values.")
}

func min(field string, value interface{}) error {

	if data, ok := value.(*int); ok {

		min := field[strings.Index(field, ":")+1:]

		if minNum, ok := strconv.ParseInt(min, 0, 64); ok == nil {

			if int64(*data) >= minNum {
				return nil
			} else {
				return errors.New("The data you passed in was smaller then the allowed minimum.")
			}

		}
	}

	return errors.New("The value passed in for MIN could not be converted to an int.")
}

func max(field string, value interface{}) error {

	if data, ok := value.(*int); ok {

		max := field[strings.Index(field, ":")+1:]

		if maxNum, ok := strconv.ParseInt(max, 0, 64); ok == nil {
			if int64(*data) <= maxNum {
				return nil
			} else {
				return errors.New("The data you passed in was larger than the maximum.")
			}

		}
	}

	return errors.New("The value passed in for MAX could not be converted to an int.")
}

// Regex handles the general regex call and also handles
// the regex email.
func regex(field string, value interface{}) error {

	reg := field[strings.Index(field, ":")+1:]

	if data, ok := value.(*string); ok {
		if len(*data) == 0 {
			return nil
		} else if err := match_regex(reg, []byte(*data)); err != nil {
			return err
		}
	} else if data, ok := value.(*int); ok {
		if err := match_regex(reg, []byte(strconv.Itoa(*data))); err != nil {
			return err
		}
	} else {
		return errors.New("The value passed in for REGEX could not be converted to a string or int.")
	}

	return nil
}

// Helper function for regex.
func match_regex(reg string, data []byte) error {

	if match, err := regexp.Match(reg, []byte(data)); err == nil && match {
		return nil
	} else {
		return errors.New("Your regex did not match or was not valid.")
	}
}

// Check passed in json length string is exact value passed in.
// Also checks if passed in values is between two different ones.
func length(field string, value interface{}) error {

	length := field[strings.Index(field, ":")+1:]

	if data, ok := value.(*string); ok {
		if intdata, intok := strconv.Atoi(length); intok == nil {
			if len(*data) == intdata {
				return nil
			} else {
				return errors.New("The data passed in was not equal to the expected length.")
			}
		} else {
			return errors.New("The value passed in for LENGTH could not be converted to an int.")
		}
	} else {
		return errors.New("The value passed in for LENGTH could not be converted to a string.")
	}
}

// Check if the strings length is between high,low.
func length_between(field string, value interface{}) error {

	length := field[strings.Index(field, ":")+1:]
	vals := strings.Split(length, ",")

	if len(vals) == 2 {

		if data, ok := value.(*string); ok {

			if lowerbound, lowok := strconv.Atoi(vals[0]); lowok == nil {

				if upperbound, upok := strconv.Atoi(vals[1]); upok == nil {

					if lowerbound <= len(*data) && upperbound >= len(*data) {
						return nil
					} else {
						return errors.New("The value passed in for LENGTH BETWEEN was not in bounds.")
					}

				} else {
					return errors.New("The value passed in for LENGTH BETWEEN could not be converted to an int.")
				}

			} else {
				return errors.New("The value passed in for LENGTH BETWEEN could not be converted to an int.")
			}

		} else {
			return errors.New("The value passed in for LENGTH BETWEEN could not be converted to a string.")
		}
	} else {
		return errors.New("LENGTH BETWEEN requires exactly two paramaters.")
	}
}
<|endoftext|>


<commit_before>package format

import (
	"bytes"
	"io"
	"os/exec"
	"sort"
	"time"

	"github.com/AlexanderThaller/lablog/src/data"
	"github.com/AlexanderThaller/lablog/src/helper"
	"github.com/AlexanderThaller/logger"
	"github.com/juju/errgo"
)

const (
	Name = "format"
)

const AsciiDocSettings = `:toc: right
:toclevels: 2
:sectanchors:
:sectlink:
:icons: font
:linkattrs:
:numbered:
:idprefix:
:idseparator: -
:doctype: book
:source-highlighter: pygments
:listing-caption: Listing`

func ProjectsEntries(writer io.Writer, projects []data.Project, start, end time.Time) error {
	io.WriteString(writer, AsciiDocSettings+"\n\n")
	io.WriteString(writer, "= Entries \n\n")

	for _, project := range projects {
		notes, err := helper.FilteredNotesByStartEnd(project, start, end)
		if err != nil {
			return errgo.Notef(err, "can not get filtered notes")
		}

		todos, err := helper.FilteredTodosByStartEnd(project, start, end)
		if err != nil {
			return errgo.Notef(err, "can not get filtered notes")
		}
		todos = data.FilterTodosLatest(todos)
		todos = data.FilterTodosAreNotDone(todos)

		project.Format(writer, 1)
		if len(todos) != 0 {
			Todos(writer, todos)
			io.WriteString(writer, "\n")
		}

		if len(notes) != 0 {
			Notes(writer, notes)
		}
	}

	return nil
}

func ProjectsNotes(writer io.Writer, projects []data.Project, start, end time.Time) error {
	l := logger.New(Name, "ProjectsNotes")

	io.WriteString(writer, AsciiDocSettings+"\n\n")
	io.WriteString(writer, "= Notes \n\n")

	for _, project := range projects {
		notes, err := helper.FilteredNotesByStartEnd(project, start, end)
		if err != nil {
			l.Debug(err)
			l.Trace(errgo.Details(errgo.Notef(err, "can not get filtered notes")))
			continue
		}

		if len(notes) == 0 {
			continue
		}

		project.Format(writer, 1)
		Notes(writer, notes)
	}

	return nil
}

func ProjectsTodos(writer io.Writer, projects []data.Project, start, end time.Time) error {
	io.WriteString(writer, AsciiDocSettings+"\n\n")
	io.WriteString(writer, "= Todos \n\n")

	for _, project := range projects {
		todos, err := helper.FilteredTodosByStartEnd(project, start, end)
		if err != nil {
			return errgo.Notef(err, "can not get filtered notes")
		}
		todos = data.FilterTodosLatest(todos)
		todos = data.FilterTodosAreNotDone(todos)

		if len(todos) == 0 {
			continue
		}

		project.Format(writer, 1)
		Todos(writer, todos)
		io.WriteString(writer, "\n")
	}

	return nil
}

func ProjectsTracks(writer io.Writer, projects []data.Project, start, end time.Time) error {
	io.WriteString(writer, AsciiDocSettings+"\n\n")
	io.WriteString(writer, "= Tracks \n\n")

	for _, project := range projects {
		tracks, err := helper.FilteredTracksByStartEnd(project, start, end)
		if err != nil {
			return errgo.Notef(err, "can not get filtered tracks")
		}

		if len(tracks) == 0 {
			continue
		}

		project.Format(writer, 1)
		Tracks(writer, tracks)
		io.WriteString(writer, "\n")
	}

	return nil
}

func ProjectsDates(writer io.Writer, projects []data.Project, start, end time.Time) error {
	io.WriteString(writer, AsciiDocSettings+"\n\n")
	io.WriteString(writer, "= Dates \n\n")

	dates, err := helper.ProjectsDates(projects, start, end)
	if err != nil {
		return errgo.Notef(err, "can not get dates for projects")
	}

	sort.Strings(dates)

	for _, date := range dates {
		io.WriteString(writer, "* "+date+"\n")
	}

	return nil
}

func Todos(writer io.Writer, todos []data.Todo) {
	io.WriteString(writer, "=== Todos\n\n")

	sort.Sort(data.TodosByName(todos))
	for _, todo := range todos {
		todo.Format(writer, 2)
	}
}

func Notes(writer io.Writer, notes []data.Note) {
	io.WriteString(writer, "=== Notes\n\n")

	notes = data.FilterNotesNotEmpty(notes)

	sort.Sort(data.NotesByTimeStamp(notes))
	for _, note := range notes {
		note.Format(writer, 2)
	}
}

func Tracks(writer io.Writer, tracks []data.Track) {
	io.WriteString(writer, "=== Tracks\n\n")

	sort.Sort(data.TracksByTimeStamp(tracks))
	for _, track := range tracks {
		track.Format(writer, 2)
	}
}

func AsciiDoctor(reader io.Reader, writer io.Writer) error {
	stderr := new(bytes.Buffer)

	command := exec.Command("asciidoctor", "-")
	command.Stdin = reader
	command.Stdout = writer
	command.Stderr = stderr

	err := command.Run()
	if err != nil {
		return errgo.Notef(errgo.Notef(err, "can not run asciidoctor"),
			stderr.String())
	}

	return nil
}

func Timeline(writer io.Writer, projects []data.Project, start, end time.Time) error {
	var allnotes []data.Note

	for _, project := range projects {
		notes, err := helper.FilteredNotesByStartEnd(project, start, end)
		if err != nil {
			return errgo.Notef(err, "can not get filtered notes")
		}

		for _, note := range notes {
			allnotes = append(allnotes, note)
		}
	}

	allnotes = data.FilterNotesNotEmpty(allnotes)
	sort.Sort(data.NotesByTimeStamp(allnotes))

	io.WriteString(writer, AsciiDocSettings+"\n\n")
	io.WriteString(writer, "= Timeline \n\n")
	for _, note := range allnotes {
		note.FormatTimeStamp(writer, 2)
		note.Project.Format(writer, 2)
		note.FormatText(writer, 4)
	}

	return nil
}
<commit_msg>also fixed subproject listing for todos and tracks.<commit_after>package format

import (
	"bytes"
	"io"
	"os/exec"
	"sort"
	"time"

	"github.com/AlexanderThaller/lablog/src/data"
	"github.com/AlexanderThaller/lablog/src/helper"
	"github.com/AlexanderThaller/logger"
	"github.com/juju/errgo"
)

const (
	Name = "format"
)

const AsciiDocSettings = `:toc: right
:toclevels: 2
:sectanchors:
:sectlink:
:icons: font
:linkattrs:
:numbered:
:idprefix:
:idseparator: -
:doctype: book
:source-highlighter: pygments
:listing-caption: Listing`

func ProjectsEntries(writer io.Writer, projects []data.Project, start, end time.Time) error {
	io.WriteString(writer, AsciiDocSettings+"\n\n")
	io.WriteString(writer, "= Entries \n\n")

	for _, project := range projects {
		notes, err := helper.FilteredNotesByStartEnd(project, start, end)
		if err != nil {
			return errgo.Notef(err, "can not get filtered notes")
		}

		todos, err := helper.FilteredTodosByStartEnd(project, start, end)
		if err != nil {
			return errgo.Notef(err, "can not get filtered notes")
		}
		todos = data.FilterTodosLatest(todos)
		todos = data.FilterTodosAreNotDone(todos)

		project.Format(writer, 1)
		if len(todos) != 0 {
			Todos(writer, todos)
			io.WriteString(writer, "\n")
		}

		if len(notes) != 0 {
			Notes(writer, notes)
		}
	}

	return nil
}

func ProjectsNotes(writer io.Writer, projects []data.Project, start, end time.Time) error {
	l := logger.New(Name, "ProjectsNotes")

	io.WriteString(writer, AsciiDocSettings+"\n\n")
	io.WriteString(writer, "= Notes \n\n")

	for _, project := range projects {
		notes, err := helper.FilteredNotesByStartEnd(project, start, end)
		if err != nil {
			l.Debug(err)
			l.Trace(errgo.Details(errgo.Notef(err, "can not get filtered notes")))
			continue
		}

		if len(notes) == 0 {
			continue
		}

		project.Format(writer, 1)
		Notes(writer, notes)
	}

	return nil
}

func ProjectsTodos(writer io.Writer, projects []data.Project, start, end time.Time) error {
	l := logger.New(Name, "ProjectTodos")

	io.WriteString(writer, AsciiDocSettings+"\n\n")
	io.WriteString(writer, "= Todos \n\n")

	for _, project := range projects {
		todos, err := helper.FilteredTodosByStartEnd(project, start, end)
		if err != nil {
			l.Debug(err)
			l.Trace(errgo.Details(errgo.Notef(err, "can not get filtered todos")))
			continue
		}
		todos = data.FilterTodosLatest(todos)
		todos = data.FilterTodosAreNotDone(todos)

		if len(todos) == 0 {
			continue
		}

		project.Format(writer, 1)
		Todos(writer, todos)
		io.WriteString(writer, "\n")
	}

	return nil
}

func ProjectsTracks(writer io.Writer, projects []data.Project, start, end time.Time) error {
	l := logger.New(Name, "ProjectTodos")

	io.WriteString(writer, AsciiDocSettings+"\n\n")
	io.WriteString(writer, "= Tracks \n\n")

	for _, project := range projects {
		tracks, err := helper.FilteredTracksByStartEnd(project, start, end)
		if err != nil {
			l.Debug(err)
			l.Trace(errgo.Details(errgo.Notef(err, "can not get filtered tracks")))
			continue
		}

		if len(tracks) == 0 {
			continue
		}

		project.Format(writer, 1)
		Tracks(writer, tracks)
		io.WriteString(writer, "\n")
	}

	return nil
}

func ProjectsDates(writer io.Writer, projects []data.Project, start, end time.Time) error {
	io.WriteString(writer, AsciiDocSettings+"\n\n")
	io.WriteString(writer, "= Dates \n\n")

	dates, err := helper.ProjectsDates(projects, start, end)
	if err != nil {
		return errgo.Notef(err, "can not get dates for projects")
	}

	sort.Strings(dates)

	for _, date := range dates {
		io.WriteString(writer, "* "+date+"\n")
	}

	return nil
}

func Todos(writer io.Writer, todos []data.Todo) {
	io.WriteString(writer, "=== Todos\n\n")

	sort.Sort(data.TodosByName(todos))
	for _, todo := range todos {
		todo.Format(writer, 2)
	}
}

func Notes(writer io.Writer, notes []data.Note) {
	io.WriteString(writer, "=== Notes\n\n")

	notes = data.FilterNotesNotEmpty(notes)

	sort.Sort(data.NotesByTimeStamp(notes))
	for _, note := range notes {
		note.Format(writer, 2)
	}
}

func Tracks(writer io.Writer, tracks []data.Track) {
	io.WriteString(writer, "=== Tracks\n\n")

	sort.Sort(data.TracksByTimeStamp(tracks))
	for _, track := range tracks {
		track.Format(writer, 2)
	}
}

func AsciiDoctor(reader io.Reader, writer io.Writer) error {
	stderr := new(bytes.Buffer)

	command := exec.Command("asciidoctor", "-")
	command.Stdin = reader
	command.Stdout = writer
	command.Stderr = stderr

	err := command.Run()
	if err != nil {
		return errgo.Notef(errgo.Notef(err, "can not run asciidoctor"),
			stderr.String())
	}

	return nil
}

func Timeline(writer io.Writer, projects []data.Project, start, end time.Time) error {
	var allnotes []data.Note

	for _, project := range projects {
		notes, err := helper.FilteredNotesByStartEnd(project, start, end)
		if err != nil {
			return errgo.Notef(err, "can not get filtered notes")
		}

		for _, note := range notes {
			allnotes = append(allnotes, note)
		}
	}

	allnotes = data.FilterNotesNotEmpty(allnotes)
	sort.Sort(data.NotesByTimeStamp(allnotes))

	io.WriteString(writer, AsciiDocSettings+"\n\n")
	io.WriteString(writer, "= Timeline \n\n")
	for _, note := range allnotes {
		note.FormatTimeStamp(writer, 2)
		note.Project.Format(writer, 2)
		note.FormatText(writer, 4)
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package helpers

import (
	"fmt"
	"time"
)

var (
	// HelperTimeout is a predefined timeout value for commands.
	HelperTimeout time.Duration = 300 // WithTimeout helper translates it to seconds
)

const (
	// BasePath is the path in the Vagrant VMs to which the test directory
	// is mounted
	BasePath = "/vagrant/"

	// ManifestBase tells ginkgo suite where to look for manifests
	K8sManifestBase = "k8sT/manifests"

	// VM / Test suite constants.
	K8s     = "k8s"
	K8s1    = "k8s1"
	K8s2    = "k8s2"
	Runtime = "runtime"

	Enabled  = "enabled"
	Disabled = "disabled"
	Total    = "total"
	Public   = "public"
	Private  = "private"

	// PolicyEnforcement represents the PolicyEnforcement configuration option
	// for the Cilium agent.
	PolicyEnforcement = "PolicyEnforcement"

	// PolicyEnforcementDefault represents the default PolicyEnforcement mode
	// for Cilium.
	PolicyEnforcementDefault = "default"

	// PolicyEnforcementAlways represents the PolicyEnforcement mode
	// for Cilium in which traffic is denied by default even when no policy
	// is imported.
	PolicyEnforcementAlways = "always"

	// PolicyEnforcementNever represents the PolicyEnforcement mode
	// for Cilium in which traffic is always allowed even if there is a policy
	// selecting endpoints.
	PolicyEnforcementNever = "never"

	// Docker Image names

	// CiliumDockerNetwork is the name of the Docker network which Cilium manages.
	CiliumDockerNetwork = "cilium-net"

	// NetperfImage is the Docker image used for performance testing
	NetperfImage = "tgraf/netperf"

	// HttpdImage is the image used for starting an HTTP server.
	HttpdImage = "cilium/demo-httpd"

	// Names of commonly used containers in tests.

	Httpd1 = "httpd1"
	Httpd2 = "httpd2"
	Httpd3 = "httpd3"
	App1   = "app1"
	App2   = "app2"
	App3   = "app3"
	Client = "client"
	Server = "server"
	Host   = "host"

	// Container lifecycle actions.
	Create = "create"
	Delete = "delete"

	// IP Address families.
	IPv4 = "IPv4"
	IPv6 = "IPv6"

	// Configuration options for endpoints. Copied from endpoint/endpoint.go
	// TODO: these should be converted into types for use in configuration
	// functions instead of using basic strings.

	OptionAllowToHost         = "AllowToHost"
	OptionAllowToWorld        = "AllowToWorld"
	OptionConntrackAccounting = "ConntrackAccounting"
	OptionConntrackLocal      = "ConntrackLocal"
	OptionConntrack           = "Conntrack"
	OptionDebug               = "Debug"
	OptionDropNotify          = "DropNotification"
	OptionTraceNotify         = "TraceNotification"
	OptionNAT46               = "NAT46"
	OptionIngressPolicy       = "IngressPolicy"
	OptionEgressPolicy        = "EgressPolicy"
	OptionIngress             = "ingress"
	OptionEgress              = "egress"
	OptionNone                = "none"
	OptionDisabled            = "Disabled"
	OptionEnabled             = "Enabled"

	PingCount          = 5
	CurlConnectTimeout = 5

	DefaultNamespace    = "default"
	KubeSystemNamespace = "kube-system"

	TestResultsPath = "test_results/"
	RunDir          = "/var/run/cilium"
	LibDir          = "/var/lib/cilium"

	DaemonName             = "cilium"
	CiliumDockerDaemonName = "cilium-docker"
	AgentDaemon            = "cilium-agent"

	GeneratedHTMLManifest   = "html.yaml"
	GeneratedServerManifest = "server.yaml"
	GeneratedClientManifest = "client.yaml"

	KubectlCreate = ResourceLifeCycleAction("create")
	KubectlDelete = ResourceLifeCycleAction("delete")
	KubectlApply  = ResourceLifeCycleAction("apply")
)

var ciliumCLICommands = map[string]string{
	"cilium endpoint list -o json":   "endpoint_list.txt",
	"cilium service list -o json":    "service_list.txt",
	"cilium config":                  "config.txt",
	"sudo cilium bpf lb list":        "bpf_lb_list.txt",
	"sudo cilium bpf ct list global": "bpf_ct_list.txt",
	"sudo cilium bpf tunnel list":    "bpf_tunnel_list.txt",
	"cilium policy get":              "policy_get.txt",
	"cilium status":                  "status.txt",
}

// ciliumKubCLICommands these commands are the same as `ciliumCLICommands` but
// it'll run inside a container and it does not have sudo support
var ciliumKubCLICommands = map[string]string{
	"cilium endpoint list -o json": "endpoint_list.txt",
	"cilium service list -o json":  "service_list.txt",
	"cilium config":                "config.txt",
	"cilium bpf lb list":           "bpf_lb_list.txt",
	"cilium bpf ct list global":    "bpf_ct_list.txt",
	"cilium bpf tunnel list":       "bpf_tunnel_list.txt",
	"cilium policy get":            "policy_get.txt",
	"cilium status":                "status.txt",
}

//GetFilePath returns the absolute path of the provided filename
func GetFilePath(filename string) string {
	return fmt.Sprintf("%s%s", BasePath, filename)
}

// K8s1VMName is the name of the Kubernetes master node when running K8s tests.
func K8s1VMName() string {
	return fmt.Sprintf("k8s1-%s", GetCurrentK8SEnv())
}

// K8s2VMName is the name of the Kubernetes worker node when running K8s tests.
func K8s2VMName() string {
	return fmt.Sprintf("k8s2-%s", GetCurrentK8SEnv())
}
<commit_msg>Ginkgo: Fix issues with link<commit_after>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package helpers

import (
	"fmt"
	"time"
)

var (
	// HelperTimeout is a predefined timeout value for commands.
	HelperTimeout time.Duration = 300 // WithTimeout helper translates it to seconds
)

const (
	// BasePath is the path in the Vagrant VMs to which the test directory
	// is mounted
	BasePath = "/src/test/"

	// ManifestBase tells ginkgo suite where to look for manifests
	K8sManifestBase = "k8sT/manifests"

	// VM / Test suite constants.
	K8s     = "k8s"
	K8s1    = "k8s1"
	K8s2    = "k8s2"
	Runtime = "runtime"

	Enabled  = "enabled"
	Disabled = "disabled"
	Total    = "total"
	Public   = "public"
	Private  = "private"

	// PolicyEnforcement represents the PolicyEnforcement configuration option
	// for the Cilium agent.
	PolicyEnforcement = "PolicyEnforcement"

	// PolicyEnforcementDefault represents the default PolicyEnforcement mode
	// for Cilium.
	PolicyEnforcementDefault = "default"

	// PolicyEnforcementAlways represents the PolicyEnforcement mode
	// for Cilium in which traffic is denied by default even when no policy
	// is imported.
	PolicyEnforcementAlways = "always"

	// PolicyEnforcementNever represents the PolicyEnforcement mode
	// for Cilium in which traffic is always allowed even if there is a policy
	// selecting endpoints.
	PolicyEnforcementNever = "never"

	// Docker Image names

	// CiliumDockerNetwork is the name of the Docker network which Cilium manages.
	CiliumDockerNetwork = "cilium-net"

	// NetperfImage is the Docker image used for performance testing
	NetperfImage = "tgraf/netperf"

	// HttpdImage is the image used for starting an HTTP server.
	HttpdImage = "cilium/demo-httpd"

	// Names of commonly used containers in tests.

	Httpd1 = "httpd1"
	Httpd2 = "httpd2"
	Httpd3 = "httpd3"
	App1   = "app1"
	App2   = "app2"
	App3   = "app3"
	Client = "client"
	Server = "server"
	Host   = "host"

	// Container lifecycle actions.
	Create = "create"
	Delete = "delete"

	// IP Address families.
	IPv4 = "IPv4"
	IPv6 = "IPv6"

	// Configuration options for endpoints. Copied from endpoint/endpoint.go
	// TODO: these should be converted into types for use in configuration
	// functions instead of using basic strings.

	OptionAllowToHost         = "AllowToHost"
	OptionAllowToWorld        = "AllowToWorld"
	OptionConntrackAccounting = "ConntrackAccounting"
	OptionConntrackLocal      = "ConntrackLocal"
	OptionConntrack           = "Conntrack"
	OptionDebug               = "Debug"
	OptionDropNotify          = "DropNotification"
	OptionTraceNotify         = "TraceNotification"
	OptionNAT46               = "NAT46"
	OptionIngressPolicy       = "IngressPolicy"
	OptionEgressPolicy        = "EgressPolicy"
	OptionIngress             = "ingress"
	OptionEgress              = "egress"
	OptionNone                = "none"
	OptionDisabled            = "Disabled"
	OptionEnabled             = "Enabled"

	PingCount          = 5
	CurlConnectTimeout = 5

	DefaultNamespace    = "default"
	KubeSystemNamespace = "kube-system"

	TestResultsPath = "test_results/"
	RunDir          = "/var/run/cilium"
	LibDir          = "/var/lib/cilium"

	DaemonName             = "cilium"
	CiliumDockerDaemonName = "cilium-docker"
	AgentDaemon            = "cilium-agent"

	GeneratedHTMLManifest   = "html.yaml"
	GeneratedServerManifest = "server.yaml"
	GeneratedClientManifest = "client.yaml"

	KubectlCreate = ResourceLifeCycleAction("create")
	KubectlDelete = ResourceLifeCycleAction("delete")
	KubectlApply  = ResourceLifeCycleAction("apply")
)

var ciliumCLICommands = map[string]string{
	"cilium endpoint list -o json":   "endpoint_list.txt",
	"cilium service list -o json":    "service_list.txt",
	"cilium config":                  "config.txt",
	"sudo cilium bpf lb list":        "bpf_lb_list.txt",
	"sudo cilium bpf ct list global": "bpf_ct_list.txt",
	"sudo cilium bpf tunnel list":    "bpf_tunnel_list.txt",
	"cilium policy get":              "policy_get.txt",
	"cilium status":                  "status.txt",
}

// ciliumKubCLICommands these commands are the same as `ciliumCLICommands` but
// it'll run inside a container and it does not have sudo support
var ciliumKubCLICommands = map[string]string{
	"cilium endpoint list -o json": "endpoint_list.txt",
	"cilium service list -o json":  "service_list.txt",
	"cilium config":                "config.txt",
	"cilium bpf lb list":           "bpf_lb_list.txt",
	"cilium bpf ct list global":    "bpf_ct_list.txt",
	"cilium bpf tunnel list":       "bpf_tunnel_list.txt",
	"cilium policy get":            "policy_get.txt",
	"cilium status":                "status.txt",
}

//GetFilePath returns the absolute path of the provided filename
func GetFilePath(filename string) string {
	return fmt.Sprintf("%s%s", BasePath, filename)
}

// K8s1VMName is the name of the Kubernetes master node when running K8s tests.
func K8s1VMName() string {
	return fmt.Sprintf("k8s1-%s", GetCurrentK8SEnv())
}

// K8s2VMName is the name of the Kubernetes worker node when running K8s tests.
func K8s2VMName() string {
	return fmt.Sprintf("k8s2-%s", GetCurrentK8SEnv())
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"os"
	"os/exec"
	"strings"
)

type Notifier struct{}

func (n *Notifier) Send(message, title, subtitle string) (err error) {
	cmd := exec.Command("osascript")

	if err = pipeAll(cmd, os.Stdout, os.Stderr); err != nil {
		return
	}

	stdin, err := cmd.StdinPipe()
	if err != nil {
		return
	}

	if err = cmd.Start(); err != nil {
		return
	}
	defer cmd.Wait()

	_, err = io.WriteString(stdin, "display notification"+escape(message)+getOptions(title, subtitle))
	if err != nil {
		return
	}

	if err = stdin.Close(); err != nil {
		return
	}

	return err
}

func escape(s string) string {
	return "\"" + strings.Replace(s, "\"", "\\\"", -1) + "\""
}

func getOptions(title, subtitle string) string {
	options := make([]string, 3, 5)
	options = append(options, "with", "title", escape(title))
	if subtitle != "" {
		options = append(options, "subtitle", subtitle)
	}
	return strings.Join(options, " ")
}

func pipeAll(cmd *exec.Cmd, stdout, stderr io.WriteCloser) (err error) {
	if err = pipe(cmd.StdoutPipe, stdout); err != nil {
		return
	}
	if err = pipe(cmd.StderrPipe, stderr); err != nil {
		return
	}
	return
}

func pipe(sourceGetter func() (io.ReadCloser, error), dist io.WriteCloser) (err error) {
	out, err := sourceGetter()
	if err != nil {
		return
	}
	go io.Copy(dist, out)
	return
}
<commit_msg>fix a bug<commit_after>package main

import (
	"io"
	"os"
	"os/exec"
	"strings"
)

type Notifier struct{}

func (n *Notifier) Send(message, title, subtitle string) (err error) {
	cmd := exec.Command("osascript")

	if err = pipeAll(cmd, os.Stdout, os.Stderr); err != nil {
		return
	}

	stdin, err := cmd.StdinPipe()
	if err != nil {
		return
	}

	if err = cmd.Start(); err != nil {
		return
	}
	defer cmd.Wait()

	_, err = io.WriteString(stdin, strings.Join([]string{"display notification", escape(message), getOptions(title, subtitle)}, " "))
	if err != nil {
		return
	}

	if err = stdin.Close(); err != nil {
		return
	}

	return err
}

func escape(s string) string {
	return "\"" + strings.Replace(s, "\"", "\\\"", -1) + "\""
}

func getOptions(title, subtitle string) string {
	options := make([]string, 0, 5)
	options = append(options, "with", "title", escape(title))
	if subtitle != "" {
		options = append(options, "subtitle", escape(subtitle))
	}
	return strings.Join(options, " ")
}

func pipeAll(cmd *exec.Cmd, stdout, stderr io.WriteCloser) (err error) {
	if err = pipe(cmd.StdoutPipe, stdout); err != nil {
		return
	}
	if err = pipe(cmd.StderrPipe, stderr); err != nil {
		return
	}
	return
}

func pipe(sourceGetter func() (io.ReadCloser, error), dist io.WriteCloser) (err error) {
	out, err := sourceGetter()
	if err != nil {
		return
	}
	go io.Copy(dist, out)
	return
}
<|endoftext|>


<commit_before>// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package model

import (
	"context"
	"errors"
	"fmt"
	"math/rand"
	"time"

	"github.com/syncthing/syncthing/lib/config"
	"github.com/syncthing/syncthing/lib/db"
	"github.com/syncthing/syncthing/lib/events"
	"github.com/syncthing/syncthing/lib/ignore"
	"github.com/syncthing/syncthing/lib/protocol"
	"github.com/syncthing/syncthing/lib/sync"
	"github.com/syncthing/syncthing/lib/watchaggregator"
)

var errWatchNotStarted = errors.New("not started")

type folder struct {
	stateTracker
	config.FolderConfiguration

	model   *Model
	shortID protocol.ShortID
	ctx     context.Context
	cancel  context.CancelFunc

	scanInterval        time.Duration
	scanTimer           *time.Timer
	scanNow             chan rescanRequest
	scanDelay           chan time.Duration
	initialScanFinished chan struct{}
	stopped             chan struct{}

	pullScheduled chan struct{}

	watchCancel      context.CancelFunc
	watchChan        chan []string
	restartWatchChan chan struct{}
	watchErr         error
	watchErrMut      sync.Mutex

	puller puller
}

type rescanRequest struct {
	subdirs []string
	err     chan error
}

type puller interface {
	pull() bool // true when successfull and should not be retried
}

func newFolder(model *Model, cfg config.FolderConfiguration) folder {
	ctx, cancel := context.WithCancel(context.Background())

	return folder{
		stateTracker:        newStateTracker(cfg.ID),
		FolderConfiguration: cfg,

		model:   model,
		shortID: model.shortID,
		ctx:     ctx,
		cancel:  cancel,

		scanInterval:        time.Duration(cfg.RescanIntervalS) * time.Second,
		scanTimer:           time.NewTimer(time.Millisecond), // The first scan should be done immediately.
		scanNow:             make(chan rescanRequest),
		scanDelay:           make(chan time.Duration),
		initialScanFinished: make(chan struct{}),
		stopped:             make(chan struct{}),

		pullScheduled: make(chan struct{}, 1), // This needs to be 1-buffered so that we queue a pull if we're busy when it comes.

		watchCancel:      func() {},
		restartWatchChan: make(chan struct{}, 1),
		watchErr:         errWatchNotStarted,
		watchErrMut:      sync.NewMutex(),
	}
}

func (f *folder) Serve() {
	l.Debugln(f, "starting")
	defer l.Debugln(f, "exiting")

	defer func() {
		f.scanTimer.Stop()
		f.setState(FolderIdle)
		close(f.stopped)
	}()

	pause := f.basePause()
	pullFailTimer := time.NewTimer(0)
	<-pullFailTimer.C

	if f.FSWatcherEnabled && f.CheckHealth() == nil {
		f.startWatch()
	}

	initialCompleted := f.initialScanFinished

	for {
		select {
		case <-f.ctx.Done():
			return

		case <-f.pullScheduled:
			pullFailTimer.Stop()
			select {
			case <-pullFailTimer.C:
			default:
			}

			if !f.puller.pull() {
				// Pulling failed, try again later.
				pullFailTimer.Reset(pause)
			}

		case <-pullFailTimer.C:
			if f.puller.pull() {
				// We're good. Don't schedule another fail pull and reset
				// the pause interval.
				pause = f.basePause()
				continue
			}

			// Pulling failed, try again later.
			l.Infof("Folder %v isn't making sync progress - retrying in %v.", f.Description(), pause)
			pullFailTimer.Reset(pause)
			// Back off from retrying to pull with an upper limit.
			if pause < 60*f.basePause() {
				pause *= 2
			}

		case <-initialCompleted:
			// Initial scan has completed, we should do a pull
			initialCompleted = nil // never hit this case again
			if !f.puller.pull() {
				// Pulling failed, try again later.
				pullFailTimer.Reset(pause)
			}

		// The reason for running the scanner from within the puller is that
		// this is the easiest way to make sure we are not doing both at the
		// same time.
		case <-f.scanTimer.C:
			l.Debugln(f, "Scanning subdirectories")
			f.scanTimerFired()

		case req := <-f.scanNow:
			req.err <- f.scanSubdirs(req.subdirs)

		case next := <-f.scanDelay:
			f.scanTimer.Reset(next)

		case fsEvents := <-f.watchChan:
			l.Debugln(f, "filesystem notification rescan")
			f.scanSubdirs(fsEvents)

		case <-f.restartWatchChan:
			f.restartWatch()
		}
	}
}

func (f *folder) BringToFront(string) {}

func (f *folder) Override(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {}

func (f *folder) DelayScan(next time.Duration) {
	f.Delay(next)
}

func (f *folder) IgnoresUpdated() {
	if f.FSWatcherEnabled {
		f.scheduleWatchRestart()
	}
}

func (f *folder) SchedulePull() {
	select {
	case f.pullScheduled <- struct{}{}:
	default:
		// We might be busy doing a pull and thus not reading from this
		// channel. The channel is 1-buffered, so one notification will be
		// queued to ensure we recheck after the pull, but beyond that we must
		// make sure to not block index receiving.
	}
}

func (f *folder) Jobs() ([]string, []string) {
	return nil, nil
}

func (f *folder) Scan(subdirs []string) error {
	<-f.initialScanFinished
	req := rescanRequest{
		subdirs: subdirs,
		err:     make(chan error),
	}
	f.scanNow <- req
	return <-req.err
}

func (f *folder) Reschedule() {
	if f.scanInterval == 0 {
		return
	}
	// Sleep a random time between 3/4 and 5/4 of the configured interval.
	sleepNanos := (f.scanInterval.Nanoseconds()*3 + rand.Int63n(2*f.scanInterval.Nanoseconds())) / 4
	interval := time.Duration(sleepNanos) * time.Nanosecond
	l.Debugln(f, "next rescan in", interval)
	f.scanTimer.Reset(interval)
}

func (f *folder) Delay(next time.Duration) {
	f.scanDelay <- next
}

func (f *folder) Stop() {
	f.cancel()
	<-f.stopped
}

// CheckHealth checks the folder for common errors, updates the folder state
// and returns the current folder error, or nil if the folder is healthy.
func (f *folder) CheckHealth() error {
	err := f.getHealthError()
	f.setError(err)
	return err
}

func (f *folder) getHealthError() error {
	// Check for folder errors, with the most serious and specific first and
	// generic ones like out of space on the home disk later.

	if err := f.CheckPath(); err != nil {
		return err
	}

	if err := f.CheckFreeSpace(); err != nil {
		return err
	}

	if err := f.model.cfg.CheckHomeFreeSpace(); err != nil {
		return err
	}

	return nil
}

func (f *folder) scanSubdirs(subDirs []string) error {
	if err := f.model.internalScanFolderSubdirs(f.ctx, f.folderID, subDirs); err != nil {
		// Potentially sets the error twice, once in the scanner just
		// by doing a check, and once here, if the error returned is
		// the same one as returned by CheckHealth, though
		// duplicate set is handled by setError.
		f.setError(err)
		return err
	}
	return nil
}

func (f *folder) scanTimerFired() {
	err := f.scanSubdirs(nil)

	select {
	case <-f.initialScanFinished:
	default:
		status := "Completed"
		if err != nil {
			status = "Failed"
		}
		l.Infoln(status, "initial scan of", f.Type.String(), "folder", f.Description())
		close(f.initialScanFinished)
	}

	f.Reschedule()
}

func (f *folder) WatchError() error {
	f.watchErrMut.Lock()
	defer f.watchErrMut.Unlock()
	return f.watchErr
}

// stopWatch immediately aborts watching and may be called asynchronously
func (f *folder) stopWatch() {
	f.watchCancel()
	f.watchErrMut.Lock()
	prevErr := f.watchErr
	f.watchErr = errWatchNotStarted
	f.watchErrMut.Unlock()
	if prevErr != errWatchNotStarted {
		data := map[string]interface{}{
			"folder": f.ID,
			"to":     errWatchNotStarted.Error(),
		}
		if prevErr != nil {
			data["from"] = prevErr.Error()
		}
		events.Default.Log(events.FolderWatchStateChanged, data)
	}
}

// scheduleWatchRestart makes sure watching is restarted from the main for loop
// in a folder's Serve and thus may be called asynchronously (e.g. when ignores change).
func (f *folder) scheduleWatchRestart() {
	select {
	case f.restartWatchChan <- struct{}{}:
	default:
		// We might be busy doing a pull and thus not reading from this
		// channel. The channel is 1-buffered, so one notification will be
		// queued to ensure we recheck after the pull.
	}
}

// restartWatch should only ever be called synchronously. If you want to use
// this asynchronously, you should probably use scheduleWatchRestart instead.
func (f *folder) restartWatch() {
	f.stopWatch()
	f.startWatch()
	f.scanSubdirs(nil)
}

// startWatch should only ever be called synchronously. If you want to use
// this asynchronously, you should probably use scheduleWatchRestart instead.
func (f *folder) startWatch() {
	ctx, cancel := context.WithCancel(f.ctx)
	f.model.fmut.RLock()
	ignores := f.model.folderIgnores[f.folderID]
	f.model.fmut.RUnlock()
	f.watchChan = make(chan []string)
	f.watchCancel = cancel
	go f.startWatchAsync(ctx, ignores)
}

// startWatchAsync tries to start the filesystem watching and retries every minute on failure.
// It is a convenience function that should not be used except in startWatch.
func (f *folder) startWatchAsync(ctx context.Context, ignores *ignore.Matcher) {
	timer := time.NewTimer(0)
	for {
		select {
		case <-timer.C:
			eventChan, err := f.Filesystem().Watch(".", ignores, ctx, f.IgnorePerms)
			f.watchErrMut.Lock()
			prevErr := f.watchErr
			f.watchErr = err
			f.watchErrMut.Unlock()
			if err != prevErr {
				data := map[string]interface{}{
					"folder": f.ID,
				}
				if prevErr != nil {
					data["from"] = prevErr.Error()
				}
				if err != nil {
					data["to"] = err.Error()
				}
				events.Default.Log(events.FolderWatchStateChanged, data)
			}
			if err != nil {
				if prevErr == errWatchNotStarted {
					l.Infof("Error while trying to start filesystem watcher for folder %s, trying again in 1min: %v", f.Description(), err)
				} else {
					l.Debugf("Repeat error while trying to start filesystem watcher for folder %s, trying again in 1min: %v", f.Description(), err)
				}
				timer.Reset(time.Minute)
				continue
			}
			watchaggregator.Aggregate(eventChan, f.watchChan, f.FolderConfiguration, f.model.cfg, ctx)
			l.Debugln("Started filesystem watcher for folder", f.Description())
			return
		case <-ctx.Done():
			return
		}
	}
}

func (f *folder) setError(err error) {
	_, _, oldErr := f.getState()
	if (err != nil && oldErr != nil && oldErr.Error() == err.Error()) || (err == nil && oldErr == nil) {
		return
	}

	if err != nil {
		if oldErr == nil {
			l.Warnf("Error on folder %s: %v", f.Description(), err)
		} else {
			l.Infof("Error on folder %s changed: %q -> %q", f.Description(), oldErr, err)
		}
	} else {
		l.Infoln("Cleared error on folder", f.Description())
	}

	if f.FSWatcherEnabled {
		if err != nil {
			f.stopWatch()
		} else {
			f.scheduleWatchRestart()
		}
	}

	f.stateTracker.setError(err)
}

func (f *folder) basePause() time.Duration {
	if f.PullerPauseS == 0 {
		return defaultPullerPause
	}
	return time.Duration(f.PullerPauseS) * time.Second
}

func (f *folder) String() string {
	return fmt.Sprintf("%s/%s@%p", f.Type, f.folderID, f)
}
<commit_msg>lib/model: Don't set watch error on folder creation (fixes 5005) (#5006)<commit_after>// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package model

import (
	"context"
	"errors"
	"fmt"
	"math/rand"
	"time"

	"github.com/syncthing/syncthing/lib/config"
	"github.com/syncthing/syncthing/lib/db"
	"github.com/syncthing/syncthing/lib/events"
	"github.com/syncthing/syncthing/lib/ignore"
	"github.com/syncthing/syncthing/lib/protocol"
	"github.com/syncthing/syncthing/lib/sync"
	"github.com/syncthing/syncthing/lib/watchaggregator"
)

var errWatchNotStarted = errors.New("not started")

type folder struct {
	stateTracker
	config.FolderConfiguration

	model   *Model
	shortID protocol.ShortID
	ctx     context.Context
	cancel  context.CancelFunc

	scanInterval        time.Duration
	scanTimer           *time.Timer
	scanNow             chan rescanRequest
	scanDelay           chan time.Duration
	initialScanFinished chan struct{}
	stopped             chan struct{}

	pullScheduled chan struct{}

	watchCancel      context.CancelFunc
	watchChan        chan []string
	restartWatchChan chan struct{}
	watchErr         error
	watchErrMut      sync.Mutex

	puller puller
}

type rescanRequest struct {
	subdirs []string
	err     chan error
}

type puller interface {
	pull() bool // true when successfull and should not be retried
}

func newFolder(model *Model, cfg config.FolderConfiguration) folder {
	ctx, cancel := context.WithCancel(context.Background())

	return folder{
		stateTracker:        newStateTracker(cfg.ID),
		FolderConfiguration: cfg,

		model:   model,
		shortID: model.shortID,
		ctx:     ctx,
		cancel:  cancel,

		scanInterval:        time.Duration(cfg.RescanIntervalS) * time.Second,
		scanTimer:           time.NewTimer(time.Millisecond), // The first scan should be done immediately.
		scanNow:             make(chan rescanRequest),
		scanDelay:           make(chan time.Duration),
		initialScanFinished: make(chan struct{}),
		stopped:             make(chan struct{}),

		pullScheduled: make(chan struct{}, 1), // This needs to be 1-buffered so that we queue a pull if we're busy when it comes.

		watchCancel:      func() {},
		restartWatchChan: make(chan struct{}, 1),
		watchErrMut:      sync.NewMutex(),
	}
}

func (f *folder) Serve() {
	l.Debugln(f, "starting")
	defer l.Debugln(f, "exiting")

	defer func() {
		f.scanTimer.Stop()
		f.setState(FolderIdle)
		close(f.stopped)
	}()

	pause := f.basePause()
	pullFailTimer := time.NewTimer(0)
	<-pullFailTimer.C

	if f.FSWatcherEnabled && f.CheckHealth() == nil {
		f.startWatch()
	}

	initialCompleted := f.initialScanFinished

	for {
		select {
		case <-f.ctx.Done():
			return

		case <-f.pullScheduled:
			pullFailTimer.Stop()
			select {
			case <-pullFailTimer.C:
			default:
			}

			if !f.puller.pull() {
				// Pulling failed, try again later.
				pullFailTimer.Reset(pause)
			}

		case <-pullFailTimer.C:
			if f.puller.pull() {
				// We're good. Don't schedule another fail pull and reset
				// the pause interval.
				pause = f.basePause()
				continue
			}

			// Pulling failed, try again later.
			l.Infof("Folder %v isn't making sync progress - retrying in %v.", f.Description(), pause)
			pullFailTimer.Reset(pause)
			// Back off from retrying to pull with an upper limit.
			if pause < 60*f.basePause() {
				pause *= 2
			}

		case <-initialCompleted:
			// Initial scan has completed, we should do a pull
			initialCompleted = nil // never hit this case again
			if !f.puller.pull() {
				// Pulling failed, try again later.
				pullFailTimer.Reset(pause)
			}

		// The reason for running the scanner from within the puller is that
		// this is the easiest way to make sure we are not doing both at the
		// same time.
		case <-f.scanTimer.C:
			l.Debugln(f, "Scanning subdirectories")
			f.scanTimerFired()

		case req := <-f.scanNow:
			req.err <- f.scanSubdirs(req.subdirs)

		case next := <-f.scanDelay:
			f.scanTimer.Reset(next)

		case fsEvents := <-f.watchChan:
			l.Debugln(f, "filesystem notification rescan")
			f.scanSubdirs(fsEvents)

		case <-f.restartWatchChan:
			f.restartWatch()
		}
	}
}

func (f *folder) BringToFront(string) {}

func (f *folder) Override(fs *db.FileSet, updateFn func([]protocol.FileInfo)) {}

func (f *folder) DelayScan(next time.Duration) {
	f.Delay(next)
}

func (f *folder) IgnoresUpdated() {
	if f.FSWatcherEnabled {
		f.scheduleWatchRestart()
	}
}

func (f *folder) SchedulePull() {
	select {
	case f.pullScheduled <- struct{}{}:
	default:
		// We might be busy doing a pull and thus not reading from this
		// channel. The channel is 1-buffered, so one notification will be
		// queued to ensure we recheck after the pull, but beyond that we must
		// make sure to not block index receiving.
	}
}

func (f *folder) Jobs() ([]string, []string) {
	return nil, nil
}

func (f *folder) Scan(subdirs []string) error {
	<-f.initialScanFinished
	req := rescanRequest{
		subdirs: subdirs,
		err:     make(chan error),
	}
	f.scanNow <- req
	return <-req.err
}

func (f *folder) Reschedule() {
	if f.scanInterval == 0 {
		return
	}
	// Sleep a random time between 3/4 and 5/4 of the configured interval.
	sleepNanos := (f.scanInterval.Nanoseconds()*3 + rand.Int63n(2*f.scanInterval.Nanoseconds())) / 4
	interval := time.Duration(sleepNanos) * time.Nanosecond
	l.Debugln(f, "next rescan in", interval)
	f.scanTimer.Reset(interval)
}

func (f *folder) Delay(next time.Duration) {
	f.scanDelay <- next
}

func (f *folder) Stop() {
	f.cancel()
	<-f.stopped
}

// CheckHealth checks the folder for common errors, updates the folder state
// and returns the current folder error, or nil if the folder is healthy.
func (f *folder) CheckHealth() error {
	err := f.getHealthError()
	f.setError(err)
	return err
}

func (f *folder) getHealthError() error {
	// Check for folder errors, with the most serious and specific first and
	// generic ones like out of space on the home disk later.

	if err := f.CheckPath(); err != nil {
		return err
	}

	if err := f.CheckFreeSpace(); err != nil {
		return err
	}

	if err := f.model.cfg.CheckHomeFreeSpace(); err != nil {
		return err
	}

	return nil
}

func (f *folder) scanSubdirs(subDirs []string) error {
	if err := f.model.internalScanFolderSubdirs(f.ctx, f.folderID, subDirs); err != nil {
		// Potentially sets the error twice, once in the scanner just
		// by doing a check, and once here, if the error returned is
		// the same one as returned by CheckHealth, though
		// duplicate set is handled by setError.
		f.setError(err)
		return err
	}
	return nil
}

func (f *folder) scanTimerFired() {
	err := f.scanSubdirs(nil)

	select {
	case <-f.initialScanFinished:
	default:
		status := "Completed"
		if err != nil {
			status = "Failed"
		}
		l.Infoln(status, "initial scan of", f.Type.String(), "folder", f.Description())
		close(f.initialScanFinished)
	}

	f.Reschedule()
}

func (f *folder) WatchError() error {
	f.watchErrMut.Lock()
	defer f.watchErrMut.Unlock()
	return f.watchErr
}

// stopWatch immediately aborts watching and may be called asynchronously
func (f *folder) stopWatch() {
	f.watchCancel()
	f.watchErrMut.Lock()
	prevErr := f.watchErr
	f.watchErr = errWatchNotStarted
	f.watchErrMut.Unlock()
	if prevErr != errWatchNotStarted {
		data := map[string]interface{}{
			"folder": f.ID,
			"to":     errWatchNotStarted.Error(),
		}
		if prevErr != nil {
			data["from"] = prevErr.Error()
		}
		events.Default.Log(events.FolderWatchStateChanged, data)
	}
}

// scheduleWatchRestart makes sure watching is restarted from the main for loop
// in a folder's Serve and thus may be called asynchronously (e.g. when ignores change).
func (f *folder) scheduleWatchRestart() {
	select {
	case f.restartWatchChan <- struct{}{}:
	default:
		// We might be busy doing a pull and thus not reading from this
		// channel. The channel is 1-buffered, so one notification will be
		// queued to ensure we recheck after the pull.
	}
}

// restartWatch should only ever be called synchronously. If you want to use
// this asynchronously, you should probably use scheduleWatchRestart instead.
func (f *folder) restartWatch() {
	f.stopWatch()
	f.startWatch()
	f.scanSubdirs(nil)
}

// startWatch should only ever be called synchronously. If you want to use
// this asynchronously, you should probably use scheduleWatchRestart instead.
func (f *folder) startWatch() {
	ctx, cancel := context.WithCancel(f.ctx)
	f.model.fmut.RLock()
	ignores := f.model.folderIgnores[f.folderID]
	f.model.fmut.RUnlock()
	f.watchChan = make(chan []string)
	f.watchCancel = cancel
	go f.startWatchAsync(ctx, ignores)
}

// startWatchAsync tries to start the filesystem watching and retries every minute on failure.
// It is a convenience function that should not be used except in startWatch.
func (f *folder) startWatchAsync(ctx context.Context, ignores *ignore.Matcher) {
	timer := time.NewTimer(0)
	for {
		select {
		case <-timer.C:
			eventChan, err := f.Filesystem().Watch(".", ignores, ctx, f.IgnorePerms)
			f.watchErrMut.Lock()
			prevErr := f.watchErr
			f.watchErr = err
			f.watchErrMut.Unlock()
			if err != prevErr {
				data := map[string]interface{}{
					"folder": f.ID,
				}
				if prevErr != nil {
					data["from"] = prevErr.Error()
				}
				if err != nil {
					data["to"] = err.Error()
				}
				events.Default.Log(events.FolderWatchStateChanged, data)
			}
			if err != nil {
				if prevErr == errWatchNotStarted {
					l.Infof("Error while trying to start filesystem watcher for folder %s, trying again in 1min: %v", f.Description(), err)
				} else {
					l.Debugf("Repeat error while trying to start filesystem watcher for folder %s, trying again in 1min: %v", f.Description(), err)
				}
				timer.Reset(time.Minute)
				continue
			}
			watchaggregator.Aggregate(eventChan, f.watchChan, f.FolderConfiguration, f.model.cfg, ctx)
			l.Debugln("Started filesystem watcher for folder", f.Description())
			return
		case <-ctx.Done():
			return
		}
	}
}

func (f *folder) setError(err error) {
	_, _, oldErr := f.getState()
	if (err != nil && oldErr != nil && oldErr.Error() == err.Error()) || (err == nil && oldErr == nil) {
		return
	}

	if err != nil {
		if oldErr == nil {
			l.Warnf("Error on folder %s: %v", f.Description(), err)
		} else {
			l.Infof("Error on folder %s changed: %q -> %q", f.Description(), oldErr, err)
		}
	} else {
		l.Infoln("Cleared error on folder", f.Description())
	}

	if f.FSWatcherEnabled {
		if err != nil {
			f.stopWatch()
		} else {
			f.scheduleWatchRestart()
		}
	}

	f.stateTracker.setError(err)
}

func (f *folder) basePause() time.Duration {
	if f.PullerPauseS == 0 {
		return defaultPullerPause
	}
	return time.Duration(f.PullerPauseS) * time.Second
}

func (f *folder) String() string {
	return fmt.Sprintf("%s/%s@%p", f.Type, f.folderID, f)
}
<|endoftext|>


<commit_before>// Copyright 2015-2017 Brett Vickers.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ntp

import (
	"strings"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

const (
	host  = "0.beevik-ntp.pool.ntp.org"
	refID = 0x58585858 // 'XXXX'
)

func isNil(t *testing.T, err error) bool {
	switch {
	case err == nil:
		return true
	case strings.Contains(err.Error(), "timeout"):
		// log instead of error, so test isn't failed
		t.Logf("[%s] Query timeout: %s", host, err)
		return false
	case strings.Contains(err.Error(), "kiss of death"):
		// log instead of error, so test isn't failed
		t.Logf("[%s] Query kiss of death: %s", host, err)
		return false
	default:
		// error, so test fails
		t.Errorf("[%s] Query failed: %s", host, err)
		return false
	}
}

func assertValid(t *testing.T, r *Response) {
	err := r.Validate()
	if err != nil {
		t.Errorf("[%s] Response invalid: %s\n", host, err)
	}
}

func assertInvalid(t *testing.T, r *Response) {
	err := r.Validate()
	if err == nil {
		t.Errorf("[%s] Response unexpectedly valid\n", host)
	}
}

func TestTime(t *testing.T) {
	tm, err := Time(host)
	now := time.Now()
	if isNil(t, err) {
		t.Logf("Local Time %v\n", now)
		t.Logf("~True Time %v\n", tm)
		t.Logf("Offset %v\n", tm.Sub(now))
	}
}

func TestTimeFailure(t *testing.T) {
	// Use a link-local IP address that won't have an NTP server listening
	// on it. This should return the local system's time.
	local, err := Time("169.254.122.229")
	assert.NotNil(t, err)

	now := time.Now()

	// When the NTP time query fails, it should return the system time.
	// Compare the "now" system time with the returned time. It should be
	// about the same.
	diffMinutes := now.Sub(local).Minutes()
	assert.True(t, diffMinutes > -1 && diffMinutes < 1)
}

func TestQuery(t *testing.T) {
	t.Logf("[%s] ----------------------", host)
	t.Logf("[%s] NTP protocol version %d", host, 4)

	r, err := QueryWithOptions(host, QueryOptions{Version: 4})
	if !isNil(t, err) {
		return
	}

	t.Logf("[%s]  LocalTime: %v", host, time.Now())
	t.Logf("[%s]   XmitTime: %v", host, r.Time)
	t.Logf("[%s]    RefTime: %v", host, r.ReferenceTime)
	t.Logf("[%s]        RTT: %v", host, r.RTT)
	t.Logf("[%s]     Offset: %v", host, r.ClockOffset)
	t.Logf("[%s]       Poll: %v", host, r.Poll)
	t.Logf("[%s]  Precision: %v", host, r.Precision)
	t.Logf("[%s]    Stratum: %v", host, r.Stratum)
	t.Logf("[%s]      RefID: 0x%08x", host, r.ReferenceID)
	t.Logf("[%s]  RootDelay: %v", host, r.RootDelay)
	t.Logf("[%s]   RootDisp: %v", host, r.RootDispersion)
	t.Logf("[%s]   RootDist: %v", host, r.RootDistance)
	t.Logf("[%s]   MinError: %v", host, r.MinError)
	t.Logf("[%s]       Leap: %v", host, r.Leap)

	assertValid(t, r)
}

func TestValidate(t *testing.T) {
	var m msg
	var r *Response
	m.Stratum = 1
	m.ReferenceID = refID
	m.ReferenceTime = 1 << 32
	m.Precision = -1 // 500ms

	// Zero RTT
	m.OriginTime = 1 << 32
	m.ReceiveTime = 1 << 32
	m.TransmitTime = 1 << 32
	r = parseTime(&m, 1<<32)
	assertValid(t, r)

	// Negative freshness
	m.ReferenceTime = 2 << 32
	r = parseTime(&m, 1<<32)
	assertInvalid(t, r)

	// Unfresh clock (48h)
	m.OriginTime = 2 * 86400 << 32
	m.ReceiveTime = 2 * 86400 << 32
	m.TransmitTime = 2 * 86400 << 32
	r = parseTime(&m, 2*86400<<32)
	assertInvalid(t, r)

	// Fresh clock (24h)
	m.ReferenceTime = 1 * 86400 << 32
	r = parseTime(&m, 2*86400<<32)
	assertValid(t, r)

	// Values indicating a negative RTT
	m.RootDelay = 16 << 16
	m.ReferenceTime = 1 << 32
	m.OriginTime = 20 << 32
	m.ReceiveTime = 10 << 32
	m.TransmitTime = 15 << 32
	r = parseTime(&m, 22<<32)
	assert.NotNil(t, r)
	assertValid(t, r)
	assert.Equal(t, r.RTT, 0*time.Second)
	assert.Equal(t, r.RootDistance, 8*time.Second)
}

func TestBadServerPort(t *testing.T) {
	// Not NTP port.
	tm, _, err := getTime(host, QueryOptions{Port: 9})
	assert.Nil(t, tm)
	assert.NotNil(t, err)
}

func TestTTL(t *testing.T) {
	// TTL of 1 should cause a timeout.
	tm, _, err := getTime(host, QueryOptions{TTL: 1})
	assert.Nil(t, tm)
	assert.NotNil(t, err)
}

func TestQueryTimeout(t *testing.T) {
	// Force an immediate timeout.
	tm, err := QueryWithOptions(host, QueryOptions{Version: 4, Timeout: time.Nanosecond})
	assert.Nil(t, tm)
	assert.NotNil(t, err)
}

func TestShortConversion(t *testing.T) {
	var ts ntpTimeShort

	ts = 0x00000000
	assert.Equal(t, 0*time.Nanosecond, ts.Duration())

	ts = 0x00000001
	assert.Equal(t, 15258*time.Nanosecond, ts.Duration()) // well, it's actually 15258.789, but it's good enough

	ts = 0x00008000
	assert.Equal(t, 500*time.Millisecond, ts.Duration()) // precise

	ts = 0x0000c000
	assert.Equal(t, 750*time.Millisecond, ts.Duration()) // precise

	ts = 0x0000ff80
	assert.Equal(t, time.Second-(1000000000/512)*time.Nanosecond, ts.Duration()) // last precise sub-second value

	ts = 0x00010000
	assert.Equal(t, 1000*time.Millisecond, ts.Duration()) // precise

	ts = 0x00018000
	assert.Equal(t, 1500*time.Millisecond, ts.Duration()) // precise

	ts = 0xffff0000
	assert.Equal(t, 65535*time.Second, ts.Duration()) // precise

	ts = 0xffffff80
	assert.Equal(t, 65536*time.Second-(1000000000/512)*time.Nanosecond, ts.Duration()) // last precise value
}

func TestLongConversion(t *testing.T) {
	ts := []ntpTime{0x0, 0xff800000, 0x1ff800000, 0x80000000ff800000, 0xffffffffff800000}

	for _, v := range ts {
		assert.Equal(t, v, toNtpTime(v.Time()))
	}
}

func abs(d time.Duration) time.Duration {
	switch {
	case int64(d) < 0:
		return -d
	default:
		return d
	}
}

func TestOffsetCalculation(t *testing.T) {
	now := time.Now()
	t1 := toNtpTime(now)
	t2 := toNtpTime(now.Add(20 * time.Second))
	t3 := toNtpTime(now.Add(21 * time.Second))
	t4 := toNtpTime(now.Add(5 * time.Second))

	// expectedOffset := ((T2 - T1) + (T3 - T4)) / 2
	// ((119 - 99) + (121 - 104)) / 2
	// (20 +  17) / 2
	// 37 / 2 = 18
	expectedOffset := 18 * time.Second
	offset := offset(t1, t2, t3, t4)
	assert.Equal(t, expectedOffset, offset)
}

func TestOffsetCalculationNegative(t *testing.T) {
	now := time.Now()
	t1 := toNtpTime(now.Add(101 * time.Second))
	t2 := toNtpTime(now.Add(102 * time.Second))
	t3 := toNtpTime(now.Add(103 * time.Second))
	t4 := toNtpTime(now.Add(105 * time.Second))

	// expectedOffset := ((T2 - T1) + (T3 - T4)) / 2
	// ((102 - 101) + (103 - 105)) / 2
	// (1 + -2) / 2 = -1 / 2
	expectedOffset := -time.Second / 2
	offset := offset(t1, t2, t3, t4)
	assert.Equal(t, expectedOffset, offset)
}

func TestMinError(t *testing.T) {
	start := time.Now()
	m := &msg{
		Stratum:       1,
		ReferenceID:   refID,
		ReferenceTime: toNtpTime(start),
		OriginTime:    toNtpTime(start.Add(1 * time.Second)),
		ReceiveTime:   toNtpTime(start.Add(2 * time.Second)),
		TransmitTime:  toNtpTime(start.Add(3 * time.Second)),
	}
	r := parseTime(m, toNtpTime(start.Add(4*time.Second)))
	assertValid(t, r)
	assert.Equal(t, r.MinError, time.Duration(0))

	for org := 1 * time.Second; org <= 10*time.Second; org += time.Second {
		for rec := 1 * time.Second; rec <= 10*time.Second; rec += time.Second {
			for xmt := rec; xmt <= 10*time.Second; xmt += time.Second {
				for dst := org; dst <= 10*time.Second; dst += time.Second {
					m.OriginTime = toNtpTime(start.Add(org))
					m.ReceiveTime = toNtpTime(start.Add(rec))
					m.TransmitTime = toNtpTime(start.Add(xmt))
					r = parseTime(m, toNtpTime(start.Add(dst)))
					assertValid(t, r)
					var error0, error1 time.Duration
					if org >= rec {
						error0 = org - rec
					}
					if xmt >= dst {
						error1 = xmt - dst
					}
					var minError time.Duration
					if error0 > error1 {
						minError = error0
					} else {
						minError = error1
					}
					assert.Equal(t, r.MinError, minError)
				}
			}
		}
	}
}
<commit_msg>Unit tests won't fail on kiss of death or timeout.<commit_after>// Copyright 2015-2017 Brett Vickers.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ntp

import (
	"strings"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

const (
	host  = "0.beevik-ntp.pool.ntp.org"
	refID = 0x58585858 // 'XXXX'
)

func isNil(t *testing.T, err error) bool {
	switch {
	case err == nil:
		return true
	case strings.Contains(err.Error(), "timeout"):
		// log instead of error, so test isn't failed
		t.Logf("[%s] Query timeout: %s", host, err)
		return false
	case strings.Contains(err.Error(), "kiss of death"):
		// log instead of error, so test isn't failed
		t.Logf("[%s] Query kiss of death: %s", host, err)
		return false
	default:
		// error, so test fails
		t.Errorf("[%s] Query failed: %s", host, err)
		return false
	}
}

func assertValid(t *testing.T, r *Response) {
	err := r.Validate()
	_ = isNil(t, err)
}

func assertInvalid(t *testing.T, r *Response) {
	err := r.Validate()
	if err == nil {
		t.Errorf("[%s] Response unexpectedly valid\n", host)
	}
}

func TestTime(t *testing.T) {
	tm, err := Time(host)
	now := time.Now()
	if isNil(t, err) {
		t.Logf("Local Time %v\n", now)
		t.Logf("~True Time %v\n", tm)
		t.Logf("Offset %v\n", tm.Sub(now))
	}
}

func TestTimeFailure(t *testing.T) {
	// Use a link-local IP address that won't have an NTP server listening
	// on it. This should return the local system's time.
	local, err := Time("169.254.122.229")
	assert.NotNil(t, err)

	now := time.Now()

	// When the NTP time query fails, it should return the system time.
	// Compare the "now" system time with the returned time. It should be
	// about the same.
	diffMinutes := now.Sub(local).Minutes()
	assert.True(t, diffMinutes > -1 && diffMinutes < 1)
}

func TestQuery(t *testing.T) {
	t.Logf("[%s] ----------------------", host)
	t.Logf("[%s] NTP protocol version %d", host, 4)

	r, err := QueryWithOptions(host, QueryOptions{Version: 4})
	if !isNil(t, err) {
		return
	}

	t.Logf("[%s]  LocalTime: %v", host, time.Now())
	t.Logf("[%s]   XmitTime: %v", host, r.Time)
	t.Logf("[%s]    RefTime: %v", host, r.ReferenceTime)
	t.Logf("[%s]        RTT: %v", host, r.RTT)
	t.Logf("[%s]     Offset: %v", host, r.ClockOffset)
	t.Logf("[%s]       Poll: %v", host, r.Poll)
	t.Logf("[%s]  Precision: %v", host, r.Precision)
	t.Logf("[%s]    Stratum: %v", host, r.Stratum)
	t.Logf("[%s]      RefID: 0x%08x", host, r.ReferenceID)
	t.Logf("[%s]  RootDelay: %v", host, r.RootDelay)
	t.Logf("[%s]   RootDisp: %v", host, r.RootDispersion)
	t.Logf("[%s]   RootDist: %v", host, r.RootDistance)
	t.Logf("[%s]   MinError: %v", host, r.MinError)
	t.Logf("[%s]       Leap: %v", host, r.Leap)

	assertValid(t, r)
}

func TestValidate(t *testing.T) {
	var m msg
	var r *Response
	m.Stratum = 1
	m.ReferenceID = refID
	m.ReferenceTime = 1 << 32
	m.Precision = -1 // 500ms

	// Zero RTT
	m.OriginTime = 1 << 32
	m.ReceiveTime = 1 << 32
	m.TransmitTime = 1 << 32
	r = parseTime(&m, 1<<32)
	assertValid(t, r)

	// Negative freshness
	m.ReferenceTime = 2 << 32
	r = parseTime(&m, 1<<32)
	assertInvalid(t, r)

	// Unfresh clock (48h)
	m.OriginTime = 2 * 86400 << 32
	m.ReceiveTime = 2 * 86400 << 32
	m.TransmitTime = 2 * 86400 << 32
	r = parseTime(&m, 2*86400<<32)
	assertInvalid(t, r)

	// Fresh clock (24h)
	m.ReferenceTime = 1 * 86400 << 32
	r = parseTime(&m, 2*86400<<32)
	assertValid(t, r)

	// Values indicating a negative RTT
	m.RootDelay = 16 << 16
	m.ReferenceTime = 1 << 32
	m.OriginTime = 20 << 32
	m.ReceiveTime = 10 << 32
	m.TransmitTime = 15 << 32
	r = parseTime(&m, 22<<32)
	assert.NotNil(t, r)
	assertValid(t, r)
	assert.Equal(t, r.RTT, 0*time.Second)
	assert.Equal(t, r.RootDistance, 8*time.Second)
}

func TestBadServerPort(t *testing.T) {
	// Not NTP port.
	tm, _, err := getTime(host, QueryOptions{Port: 9})
	assert.Nil(t, tm)
	assert.NotNil(t, err)
}

func TestTTL(t *testing.T) {
	// TTL of 1 should cause a timeout.
	tm, _, err := getTime(host, QueryOptions{TTL: 1})
	assert.Nil(t, tm)
	assert.NotNil(t, err)
}

func TestQueryTimeout(t *testing.T) {
	// Force an immediate timeout.
	tm, err := QueryWithOptions(host, QueryOptions{Version: 4, Timeout: time.Nanosecond})
	assert.Nil(t, tm)
	assert.NotNil(t, err)
}

func TestShortConversion(t *testing.T) {
	var ts ntpTimeShort

	ts = 0x00000000
	assert.Equal(t, 0*time.Nanosecond, ts.Duration())

	ts = 0x00000001
	assert.Equal(t, 15258*time.Nanosecond, ts.Duration()) // well, it's actually 15258.789, but it's good enough

	ts = 0x00008000
	assert.Equal(t, 500*time.Millisecond, ts.Duration()) // precise

	ts = 0x0000c000
	assert.Equal(t, 750*time.Millisecond, ts.Duration()) // precise

	ts = 0x0000ff80
	assert.Equal(t, time.Second-(1000000000/512)*time.Nanosecond, ts.Duration()) // last precise sub-second value

	ts = 0x00010000
	assert.Equal(t, 1000*time.Millisecond, ts.Duration()) // precise

	ts = 0x00018000
	assert.Equal(t, 1500*time.Millisecond, ts.Duration()) // precise

	ts = 0xffff0000
	assert.Equal(t, 65535*time.Second, ts.Duration()) // precise

	ts = 0xffffff80
	assert.Equal(t, 65536*time.Second-(1000000000/512)*time.Nanosecond, ts.Duration()) // last precise value
}

func TestLongConversion(t *testing.T) {
	ts := []ntpTime{0x0, 0xff800000, 0x1ff800000, 0x80000000ff800000, 0xffffffffff800000}

	for _, v := range ts {
		assert.Equal(t, v, toNtpTime(v.Time()))
	}
}

func abs(d time.Duration) time.Duration {
	switch {
	case int64(d) < 0:
		return -d
	default:
		return d
	}
}

func TestOffsetCalculation(t *testing.T) {
	now := time.Now()
	t1 := toNtpTime(now)
	t2 := toNtpTime(now.Add(20 * time.Second))
	t3 := toNtpTime(now.Add(21 * time.Second))
	t4 := toNtpTime(now.Add(5 * time.Second))

	// expectedOffset := ((T2 - T1) + (T3 - T4)) / 2
	// ((119 - 99) + (121 - 104)) / 2
	// (20 +  17) / 2
	// 37 / 2 = 18
	expectedOffset := 18 * time.Second
	offset := offset(t1, t2, t3, t4)
	assert.Equal(t, expectedOffset, offset)
}

func TestOffsetCalculationNegative(t *testing.T) {
	now := time.Now()
	t1 := toNtpTime(now.Add(101 * time.Second))
	t2 := toNtpTime(now.Add(102 * time.Second))
	t3 := toNtpTime(now.Add(103 * time.Second))
	t4 := toNtpTime(now.Add(105 * time.Second))

	// expectedOffset := ((T2 - T1) + (T3 - T4)) / 2
	// ((102 - 101) + (103 - 105)) / 2
	// (1 + -2) / 2 = -1 / 2
	expectedOffset := -time.Second / 2
	offset := offset(t1, t2, t3, t4)
	assert.Equal(t, expectedOffset, offset)
}

func TestMinError(t *testing.T) {
	start := time.Now()
	m := &msg{
		Stratum:       1,
		ReferenceID:   refID,
		ReferenceTime: toNtpTime(start),
		OriginTime:    toNtpTime(start.Add(1 * time.Second)),
		ReceiveTime:   toNtpTime(start.Add(2 * time.Second)),
		TransmitTime:  toNtpTime(start.Add(3 * time.Second)),
	}
	r := parseTime(m, toNtpTime(start.Add(4*time.Second)))
	assertValid(t, r)
	assert.Equal(t, r.MinError, time.Duration(0))

	for org := 1 * time.Second; org <= 10*time.Second; org += time.Second {
		for rec := 1 * time.Second; rec <= 10*time.Second; rec += time.Second {
			for xmt := rec; xmt <= 10*time.Second; xmt += time.Second {
				for dst := org; dst <= 10*time.Second; dst += time.Second {
					m.OriginTime = toNtpTime(start.Add(org))
					m.ReceiveTime = toNtpTime(start.Add(rec))
					m.TransmitTime = toNtpTime(start.Add(xmt))
					r = parseTime(m, toNtpTime(start.Add(dst)))
					assertValid(t, r)
					var error0, error1 time.Duration
					if org >= rec {
						error0 = org - rec
					}
					if xmt >= dst {
						error1 = xmt - dst
					}
					var minError time.Duration
					if error0 > error1 {
						minError = error0
					} else {
						minError = error1
					}
					assert.Equal(t, r.MinError, minError)
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package docconv

import (
	"fmt"
	"os/exec"
	"strings"
	"time"
)

// Meta data
type MetaResult struct {
	meta map[string]string
	err  error
}

type BodyResult struct {
	body string
	err  error
}

// Convert PDF

func ConvertPDFText(path string) (BodyResult, MetaResult, error) {
	metaResult := MetaResult{meta: make(map[string]string)}
	bodyResult := BodyResult{}
	mr := make(chan MetaResult, 1)
	go func() {
		metaStr, err := exec.Command("pdfinfo", path).Output()
		if err != nil {
			metaResult.err = err
			mr <- metaResult
			return
		}

		// Parse meta output
		for _, line := range strings.Split(string(metaStr), "\n") {
			if parts := strings.SplitN(line, ":", 2); len(parts) > 1 {
				metaResult.meta[strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1])
			}
		}

		// Convert parsed meta
		if tmp, ok := metaResult.meta["Author"]; ok {
			metaResult.meta["Author"] = tmp
		}
		if tmp, ok := metaResult.meta["ModDate"]; ok {
			if t, err := time.Parse(time.ANSIC, tmp); err == nil {
				metaResult.meta["ModifiedDate"] = fmt.Sprintf("%d", t.Unix())
			}
		}
		if tmp, ok := metaResult.meta["CreationDate"]; ok {
			if t, err := time.Parse(time.ANSIC, tmp); err == nil {
				metaResult.meta["CreatedDate"] = fmt.Sprintf("%d", t.Unix())
			}
		}

		mr <- metaResult
	}()

	br := make(chan BodyResult, 1)
	go func() {
		body, err := exec.Command("pdftotext", "-q", "-nopgbrk", "-enc", "UTF-8", "-eol", "unix", path, "-").Output()
		if err != nil {
			bodyResult.err = err
		}

		bodyResult.body = string(body)

		br <- bodyResult
	}()

	return <-br, <-mr, nil
}
<commit_msg>pdf: add extra time layout for pdfs<commit_after>package docconv

import (
	"fmt"
	"os/exec"
	"strings"
	"time"
)

// Meta data
type MetaResult struct {
	meta map[string]string
	err  error
}

type BodyResult struct {
	body string
	err  error
}

// Convert PDF

func ConvertPDFText(path string) (BodyResult, MetaResult, error) {
	metaResult := MetaResult{meta: make(map[string]string)}
	bodyResult := BodyResult{}
	mr := make(chan MetaResult, 1)
	go func() {
		metaStr, err := exec.Command("pdfinfo", path).Output()
		if err != nil {
			metaResult.err = err
			mr <- metaResult
			return
		}

		// Parse meta output
		for _, line := range strings.Split(string(metaStr), "\n") {
			if parts := strings.SplitN(line, ":", 2); len(parts) > 1 {
				metaResult.meta[strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1])
			}
		}

		// Convert parsed meta
		if x, ok := metaResult.meta["ModDate"]; ok {
			if t, ok := pdfTimeLayouts.Parse(x); ok {
				metaResult.meta["ModifiedDate"] = fmt.Sprintf("%d", t.Unix())
			}
		}
		if x, ok := metaResult.meta["CreationDate"]; ok {
			if t, ok := pdfTimeLayouts.Parse(x); ok {
				metaResult.meta["CreatedDate"] = fmt.Sprintf("%d", t.Unix())
			}
		}

		mr <- metaResult
	}()

	br := make(chan BodyResult, 1)
	go func() {
		body, err := exec.Command("pdftotext", "-q", "-nopgbrk", "-enc", "UTF-8", "-eol", "unix", path, "-").Output()
		if err != nil {
			bodyResult.err = err
		}

		bodyResult.body = string(body)

		br <- bodyResult
	}()

	return <-br, <-mr, nil
}

var pdfTimeLayouts = timeLayouts{time.ANSIC, "Mon Jan _2 15:04:05 2006 MST"}

type timeLayouts []string

func (tl timeLayouts) Parse(x string) (time.Time, bool) {
	for _, layout := range tl {
		t, err := time.Parse(layout, x)
		if err == nil {
			return t, true
		}
	}
	return time.Time{}, false
}
<|endoftext|>