<commit_before>// Copyright 2016 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package merkle

import (
	"fmt"
	"math/bits"

	"github.com/golang/glog"
	"github.com/google/trillian/merkle/compact"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

// Verbosity levels for logging of debug related items
const vLevel = 2
const vvLevel = 4

// NodeFetch bundles a nodeID with additional information on how to use the node to construct the
// correct proof.
type NodeFetch struct {
	ID     compact.NodeID
	Rehash bool
}

// checkSnapshot performs a couple of simple sanity checks on ss and treeSize
// and returns an error if there's a problem.
func checkSnapshot(ssDesc string, ss, treeSize int64) error {
	if ss < 1 {
		return fmt.Errorf("%s %d < 1", ssDesc, ss)
	}
	if ss > treeSize {
		return fmt.Errorf("%s %d > treeSize %d", ssDesc, ss, treeSize)
	}
	return nil
}

// CalcInclusionProofNodeAddresses returns the tree node IDs needed to build an
// inclusion proof for a specified leaf and tree size. The snapshot parameter
// is the tree size being queried for, treeSize is the actual size of the tree
// at the revision we are using to fetch nodes (this can be > snapshot).
func CalcInclusionProofNodeAddresses(snapshot, index, treeSize int64) ([]NodeFetch, error) {
	if err := checkSnapshot("snapshot", snapshot, treeSize); err != nil {
		return nil, status.Errorf(codes.InvalidArgument, "invalid parameter for inclusion proof: %v", err)
	}
	if index >= snapshot {
		return nil, status.Errorf(codes.InvalidArgument, "invalid parameter for inclusion proof: index %d is >= snapshot %d", index, snapshot)
	}
	if index < 0 {
		return nil, status.Errorf(codes.InvalidArgument, "invalid parameter for inclusion proof: index %d is < 0", index)
	}
	// Note: If snapshot < treeSize, the storage might not contain the
	// "ephemeral" node of this proof, so rehashing is needed.
	return proofNodes(uint64(index), 0, uint64(snapshot), snapshot < treeSize), nil
}

// CalcConsistencyProofNodeAddresses returns the tree node IDs needed to build
// a consistency proof between two specified tree sizes. snapshot1 and
// snapshot2 represent the two tree sizes for which consistency should be
// proved, treeSize is the actual size of the tree at the revision we are using
// to fetch nodes (this can be > snapshot2).
//
// The caller is responsible for checking that the input tree sizes correspond
// to valid tree heads. All returned NodeIDs are tree coordinates within the
// new tree. It is assumed that they will be fetched from storage at a revision
// corresponding to the STH associated with the treeSize parameter.
func CalcConsistencyProofNodeAddresses(snapshot1, snapshot2, treeSize int64) ([]NodeFetch, error) {
	if err := checkSnapshot("snapshot1", snapshot1, treeSize); err != nil {
		return nil, status.Errorf(codes.InvalidArgument, "invalid parameter for consistency proof: %v", err)
	}
	if err := checkSnapshot("snapshot2", snapshot2, treeSize); err != nil {
		return nil, status.Errorf(codes.InvalidArgument, "invalid parameter for consistency proof: %v", err)
	}
	if snapshot1 > snapshot2 {
		return nil, status.Errorf(codes.InvalidArgument, "invalid parameter for consistency proof: snapshot1 %d > snapshot2 %d", snapshot1, snapshot2)
	}

	return snapshotConsistency(snapshot1, snapshot2, treeSize)
}

// snapshotConsistency does the calculation of consistency proof node addresses between
// two snapshots. Based on the C++ code used by CT but adjusted to fit our situation.
func snapshotConsistency(snapshot1, snapshot2, treeSize int64) ([]NodeFetch, error) {
	proof := make([]NodeFetch, 0, bits.Len64(uint64(snapshot2))+1)

	glog.V(vLevel).Infof("snapshotConsistency: %d -> %d", snapshot1, snapshot2)

	if snapshot1 == snapshot2 {
		return proof, nil
	}

	level := uint(0)
	node := snapshot1 - 1

	// Compute the (compressed) path to the root of snapshot2.
	// Everything left of 'node' is equal in both trees; no need to record.
	for (node & 1) != 0 {
		glog.V(vvLevel).Infof("Move up: l:%d n:%d", level, node)
		node >>= 1
		level++
	}

	if node != 0 {
		glog.V(vvLevel).Infof("Not root snapshot1: %d", node)
		// Not at the root of snapshot 1, record the node
		n := compact.NewNodeID(level, uint64(node))
		proof = append(proof, NodeFetch{ID: n})
	}

	// Now append the path from this node to the root of snapshot2.
	p := proofNodes(uint64(node), level, uint64(snapshot2), snapshot2 < treeSize)
	return append(proof, p...), nil
}

// proofNodes returns the node IDs necessary to prove that the (level, index)
// node is included in the Merkle tree of the given size.
func proofNodes(index uint64, level uint, size uint64, rehash bool) []NodeFetch {
	// [begin, end) is the leaves range covered by the (level, index) node.
	begin, end := index<<level, (index+1)<<level
	// To prove inclusion of range [begin, end), we only need nodes of compact
	// range [0, begin) and [end, size). Further down, we need the nodes ordered
	// by level from leaves towards the root.
	left := reverse(compact.RangeNodes(0, begin))
	// We decompose the [end, size) range into [end, end+l) and [end+l, size).
	// The first one (named `middle` here) contains all the nodes that don't have
	// a left sibling within [end, size), and the second one (named `right`
	// below) contains all the nodes that don't have a right sibling.
	l, r := compact.Decompose(end, size)
	middle := compact.RangeNodes(end, end+l)

	// Nodes that don't have a right sibling (i.e. the right border of the tree)
	// are special, because their hashes are collapsed into a single "ephemeral"
	// hash. This hash is already known if rehash==false, otherwise the caller
	// needs to compute it based on the hashes of compact range [end+l, size).
	var right []compact.NodeID
	if r != 0 {
		if rehash {
			right = reverse(compact.RangeNodes(end+l, size))
			rehash = len(right) > 1
		} else {
			// The parent of the highest node in [end+l, size) is "ephemeral".
			lvl := uint(bits.Len64(r))
			// Except when [end+l, size) is a perfect subtree, in which case we just
			// take the root node.
			if r&(r-1) == 0 {
				lvl--
			}
			right = []compact.NodeID{compact.NewNodeID(lvl, (end+l)>>lvl)}
		}
	}

	// The level in the ordered list of nodes where the rehashed nodes appear in
	// lieu of the "ephemeral" node. This is equal to the level where the path to
	// the `begin` index diverges from the path to `size`.
	rehashLevel := uint(bits.Len64(begin^size) - 1)

	// Merge the three compact ranges into a single proof ordered by node level
	// from leaves towards the root, i.e. the format specified in RFC 6962.
	proof := make([]NodeFetch, 0, len(left)+len(middle)+len(right))
	i, j := 0, 0
	for l, levels := level, uint(bits.Len64(size-1)); l < levels; l++ {
		if i < len(left) && left[i].Level == l {
			proof = append(proof, NodeFetch{ID: left[i]})
			i++
		} else if j < len(middle) && middle[j].Level == l {
			proof = append(proof, NodeFetch{ID: middle[j]})
			j++
		}
		if l == rehashLevel {
			for _, id := range right {
				proof = append(proof, NodeFetch{ID: id, Rehash: rehash})
			}
		}
	}

	return proof
}

func reverse(ids []compact.NodeID) []compact.NodeID {
	for i, j := 0, len(ids)-1; i < j; i, j = i+1, j-1 {
		ids[i], ids[j] = ids[j], ids[i]
	}
	return ids
}
<commit_msg>merkle: Refactor snapshotConsistency function (#2163)<commit_after>// Copyright 2016 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package merkle

import (
	"fmt"
	"math/bits"

	"github.com/golang/glog"
	"github.com/google/trillian/merkle/compact"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

// Verbosity levels for logging of debug related items
const vLevel = 2
const vvLevel = 4

// NodeFetch bundles a nodeID with additional information on how to use the node to construct the
// correct proof.
type NodeFetch struct {
	ID     compact.NodeID
	Rehash bool
}

// checkSnapshot performs a couple of simple sanity checks on ss and treeSize
// and returns an error if there's a problem.
func checkSnapshot(ssDesc string, ss, treeSize int64) error {
	if ss < 1 {
		return fmt.Errorf("%s %d < 1", ssDesc, ss)
	}
	if ss > treeSize {
		return fmt.Errorf("%s %d > treeSize %d", ssDesc, ss, treeSize)
	}
	return nil
}

// CalcInclusionProofNodeAddresses returns the tree node IDs needed to build an
// inclusion proof for a specified leaf and tree size. The snapshot parameter
// is the tree size being queried for, treeSize is the actual size of the tree
// at the revision we are using to fetch nodes (this can be > snapshot).
func CalcInclusionProofNodeAddresses(snapshot, index, treeSize int64) ([]NodeFetch, error) {
	if err := checkSnapshot("snapshot", snapshot, treeSize); err != nil {
		return nil, status.Errorf(codes.InvalidArgument, "invalid parameter for inclusion proof: %v", err)
	}
	if index >= snapshot {
		return nil, status.Errorf(codes.InvalidArgument, "invalid parameter for inclusion proof: index %d is >= snapshot %d", index, snapshot)
	}
	if index < 0 {
		return nil, status.Errorf(codes.InvalidArgument, "invalid parameter for inclusion proof: index %d is < 0", index)
	}
	// Note: If snapshot < treeSize, the storage might not contain the
	// "ephemeral" node of this proof, so rehashing is needed.
	return proofNodes(uint64(index), 0, uint64(snapshot), snapshot < treeSize), nil
}

// CalcConsistencyProofNodeAddresses returns the tree node IDs needed to build
// a consistency proof between two specified tree sizes. snapshot1 and
// snapshot2 represent the two tree sizes for which consistency should be
// proved, treeSize is the actual size of the tree at the revision we are using
// to fetch nodes (this can be > snapshot2).
//
// The caller is responsible for checking that the input tree sizes correspond
// to valid tree heads. All returned NodeIDs are tree coordinates within the
// new tree. It is assumed that they will be fetched from storage at a revision
// corresponding to the STH associated with the treeSize parameter.
func CalcConsistencyProofNodeAddresses(snapshot1, snapshot2, treeSize int64) ([]NodeFetch, error) {
	if err := checkSnapshot("snapshot1", snapshot1, treeSize); err != nil {
		return nil, status.Errorf(codes.InvalidArgument, "invalid parameter for consistency proof: %v", err)
	}
	if err := checkSnapshot("snapshot2", snapshot2, treeSize); err != nil {
		return nil, status.Errorf(codes.InvalidArgument, "invalid parameter for consistency proof: %v", err)
	}
	if snapshot1 > snapshot2 {
		return nil, status.Errorf(codes.InvalidArgument, "invalid parameter for consistency proof: snapshot1 %d > snapshot2 %d", snapshot1, snapshot2)
	}

	return snapshotConsistency(snapshot1, snapshot2, treeSize)
}

// snapshotConsistency does the calculation of consistency proof node addresses
// between two snapshots in a bigger tree of the given size.
func snapshotConsistency(snapshot1, snapshot2, treeSize int64) ([]NodeFetch, error) {
	glog.V(vLevel).Infof("snapshotConsistency: %d -> %d", snapshot1, snapshot2)
	if snapshot1 == snapshot2 {
		return []NodeFetch{}, nil
	}

	// TODO(pavelkalinnikov): Make the capacity estimate accurate.
	proof := make([]NodeFetch, 0, bits.Len64(uint64(snapshot2))+1)

	// Find the biggest perfect subtree that ends at snapshot1.
	level := uint(bits.TrailingZeros64(uint64(snapshot1)))
	index := uint64((snapshot1 - 1)) >> level
	// If it does not cover the whole snapshot1 tree, add this node to the proof.
	if index != 0 {
		glog.V(vvLevel).Infof("Not root snapshot1: %d", index)
		n := compact.NewNodeID(level, index)
		proof = append(proof, NodeFetch{ID: n})
	}

	// Now append the path from this node to the root of snapshot2.
	p := proofNodes(index, level, uint64(snapshot2), snapshot2 < treeSize)
	return append(proof, p...), nil
}

// proofNodes returns the node IDs necessary to prove that the (level, index)
// node is included in the Merkle tree of the given size.
func proofNodes(index uint64, level uint, size uint64, rehash bool) []NodeFetch {
	// [begin, end) is the leaves range covered by the (level, index) node.
	begin, end := index<<level, (index+1)<<level
	// To prove inclusion of range [begin, end), we only need nodes of compact
	// range [0, begin) and [end, size). Further down, we need the nodes ordered
	// by level from leaves towards the root.
	left := reverse(compact.RangeNodes(0, begin))
	// We decompose the [end, size) range into [end, end+l) and [end+l, size).
	// The first one (named `middle` here) contains all the nodes that don't have
	// a left sibling within [end, size), and the second one (named `right`
	// below) contains all the nodes that don't have a right sibling.
	l, r := compact.Decompose(end, size)
	middle := compact.RangeNodes(end, end+l)

	// Nodes that don't have a right sibling (i.e. the right border of the tree)
	// are special, because their hashes are collapsed into a single "ephemeral"
	// hash. This hash is already known if rehash==false, otherwise the caller
	// needs to compute it based on the hashes of compact range [end+l, size).
	var right []compact.NodeID
	if r != 0 {
		if rehash {
			right = reverse(compact.RangeNodes(end+l, size))
			rehash = len(right) > 1
		} else {
			// The parent of the highest node in [end+l, size) is "ephemeral".
			lvl := uint(bits.Len64(r))
			// Except when [end+l, size) is a perfect subtree, in which case we just
			// take the root node.
			if r&(r-1) == 0 {
				lvl--
			}
			right = []compact.NodeID{compact.NewNodeID(lvl, (end+l)>>lvl)}
		}
	}

	// The level in the ordered list of nodes where the rehashed nodes appear in
	// lieu of the "ephemeral" node. This is equal to the level where the path to
	// the `begin` index diverges from the path to `size`.
	rehashLevel := uint(bits.Len64(begin^size) - 1)

	// Merge the three compact ranges into a single proof ordered by node level
	// from leaves towards the root, i.e. the format specified in RFC 6962.
	proof := make([]NodeFetch, 0, len(left)+len(middle)+len(right))
	i, j := 0, 0
	for l, levels := level, uint(bits.Len64(size-1)); l < levels; l++ {
		if i < len(left) && left[i].Level == l {
			proof = append(proof, NodeFetch{ID: left[i]})
			i++
		} else if j < len(middle) && middle[j].Level == l {
			proof = append(proof, NodeFetch{ID: middle[j]})
			j++
		}
		if l == rehashLevel {
			for _, id := range right {
				proof = append(proof, NodeFetch{ID: id, Rehash: rehash})
			}
		}
	}

	return proof
}

func reverse(ids []compact.NodeID) []compact.NodeID {
	for i, j := 0, len(ids)-1; i < j; i, j = i+1, j-1 {
		ids[i], ids[j] = ids[j], ids[i]
	}
	return ids
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aip0131

import (
	"strings"
	"testing"

	"github.com/golang/protobuf/proto"
	dpb "github.com/golang/protobuf/protoc-gen-go/descriptor"
	"github.com/jhump/protoreflect/desc/builder"
	"google.golang.org/genproto/googleapis/api/annotations"
)

func TestRequestMessageName(t *testing.T) {
	// Set up the testing permutations.
	tests := []struct {
		testName       string
		methodName     string
		reqMessageName string
		problemCount   int
		errPrefix      string
	}{
		{"Valid", "GetBook", "GetBookRequest", 0, "False positive"},
		{"Invalid", "GetBook", "Book", 1, "False negative"},
		{"GetIamPolicy", "GetIamPolicy", "GetIamPolicyRequest", 0, "False positive"},
		{"Irrelevant", "AcquireBook", "Book", 0, "False positive"},
	}

	// Run each test individually.
	for _, test := range tests {
		t.Run(test.testName, func(t *testing.T) {
			// Create a minimal service with a AIP-131 Get method
			// (or with a different method, in the "Irrelevant" case).
			service, err := builder.NewService("Library").AddMethod(builder.NewMethod(test.methodName,
				builder.RpcTypeMessage(builder.NewMessage(test.reqMessageName), false),
				builder.RpcTypeMessage(builder.NewMessage("Book"), false),
			)).Build()
			if err != nil {
				t.Fatalf("Could not build %s method.", test.methodName)
			}

			// Run the lint rule, and establish that it returns the correct
			// number of problems.
			if problems := requestMessageName.Lint(service.GetFile()); len(problems) != test.problemCount {
				t.Errorf("%s on rule %s: %#v", test.errPrefix, requestMessageName.Name, problems)
			}
		})
	}
}

func TestResponseMessageName(t *testing.T) {
	// Set up the testing permutations.
	tests := []struct {
		testName        string
		methodName      string
		respMessageName string
		problemCount    int
		errPrefix       string
	}{
		{"Valid", "GetBook", "Book", 0, "False positive"},
		{"Invalid", "GetBook", "GetBookResponse", 1, "False negative"},
		{"Irrelevant", "AcquireBook", "AcquireBookResponse", 0, "False positive"},
	}

	// Run each test individually.
	for _, test := range tests {
		t.Run(test.testName, func(t *testing.T) {
			// Create a minimal service with a AIP-131 Get method
			// (or with a different method, in the "Irrelevant" case).
			service, err := builder.NewService("Library").AddMethod(builder.NewMethod(test.methodName,
				builder.RpcTypeMessage(builder.NewMessage("GetBookRequest"), false),
				builder.RpcTypeMessage(builder.NewMessage(test.respMessageName), false),
			)).Build()
			if err != nil {
				t.Fatalf("Could not build %s method.", test.methodName)
			}

			// Run the lint rule, and establish that it returns the correct
			// number of problems.
			if problems := responseMessageName.Lint(service.GetFile()); len(problems) != test.problemCount {
				t.Errorf("%s on rule %s: %#v", test.errPrefix, responseMessageName.Name, problems)
			}
		})
	}
}

func TestHttpVerb(t *testing.T) {
	// Set up GET and POST HTTP annotations.
	httpGet := &annotations.HttpRule{
		Pattern: &annotations.HttpRule_Get{
			Get: "/v1/{name=publishers/*/books/*}",
		},
	}
	httpPost := &annotations.HttpRule{
		Pattern: &annotations.HttpRule_Post{
			Post: "/v1/{name=publishers/*/books/*}",
		},
	}

	// Set up testing permutations.
	tests := []struct {
		testName   string
		httpRule   *annotations.HttpRule
		methodName string
		msg        string
	}{
		{"Valid", httpGet, "GetBook", ""},
		{"Invalid", httpPost, "GetBook", "HTTP GET"},
		{"Irrelevant", httpPost, "AcquireBook", ""},
	}

	// Run each test.
	for _, test := range tests {
		t.Run(test.testName, func(t *testing.T) {
			// Create a MethodOptions with the annotation set.
			opts := &dpb.MethodOptions{}
			if err := proto.SetExtension(opts, annotations.E_Http, test.httpRule); err != nil {
				t.Fatalf("Failed to set google.api.http annotation.")
			}

			// Create a minimal service with a AIP-131 Get method
			// (or with a different method, in the "Irrelevant" case).
			service, err := builder.NewService("Library").AddMethod(builder.NewMethod(test.methodName,
				builder.RpcTypeMessage(builder.NewMessage("GetBookRequest"), false),
				builder.RpcTypeMessage(builder.NewMessage("Book"), false),
			).SetOptions(opts)).Build()
			if err != nil {
				t.Fatalf("Could not build %s method.", test.methodName)
			}

			// Run the method, ensure we get what we expect.
			problems := httpVerb.Lint(service.GetFile())
			if test.msg == "" && len(problems) > 0 {
				t.Errorf("Got %v, expected no problems.", problems)
			} else if test.msg != "" && !strings.Contains(problems[0].Message, test.msg) {
				t.Errorf("Got %q, expected message containing %q", problems[0].Message, test.msg)
			}
		})
	}
}

func TestHttpBody(t *testing.T) {
	tests := []struct {
		testName   string
		body       string
		methodName string
		msg        string
	}{
		{"Valid", "", "GetBook", ""},
		{"Invalid", "*", "GetBook", "HTTP body"},
		{"Irrelevant", "*", "AcquireBook", ""},
	}

	for _, test := range tests {
		t.Run(test.testName, func(t *testing.T) {
			// Create a MethodOptions with the annotation set.
			opts := &dpb.MethodOptions{}
			httpRule := &annotations.HttpRule{
				Pattern: &annotations.HttpRule_Get{
					Get: "/v1/{name=publishers/*/books/*}",
				},
				Body: test.body,
			}
			if err := proto.SetExtension(opts, annotations.E_Http, httpRule); err != nil {
				t.Fatalf("Failed to set google.api.http annotation.")
			}

			// Create a minimal service with a AIP-131 Get method
			// (or with a different method, in the "Irrelevant" case).
			service, err := builder.NewService("Library").AddMethod(builder.NewMethod(test.methodName,
				builder.RpcTypeMessage(builder.NewMessage("GetBookRequest"), false),
				builder.RpcTypeMessage(builder.NewMessage("Book"), false),
			).SetOptions(opts)).Build()
			if err != nil {
				t.Fatalf("Could not build %s method.", test.methodName)
			}

			// Run the method, ensure we get what we expect.
			problems := httpBody.Lint(service.GetFile())
			if test.msg == "" && len(problems) > 0 {
				t.Errorf("Got %v, expected no problems.", problems)
			} else if test.msg != "" && !strings.Contains(problems[0].Message, test.msg) {
				t.Errorf("Got %q, expected message containing %q", problems[0].Message, test.msg)
			}
		})
	}
}

func TestHttpNameField(t *testing.T) {
	tests := []struct {
		testName   string
		uri        string
		methodName string
		msg        string
	}{
		{"Valid", "/v1/{name=publishers/*/books/*}", "GetBook", ""},
		{"InvalidVarName", "/v1/{book=publishers/*/books/*}", "GetBook", "`name` field"},
		{"NoVarName", "/v1/publishers/*/books/*", "GetBook", "`name` field"},
		{"Irrelevant", "/v1/{book=publishers/*/books/*}", "AcquireBook", ""},
	}

	for _, test := range tests {
		t.Run(test.testName, func(t *testing.T) {
			// Create a MethodOptions with the annotation set.
			opts := &dpb.MethodOptions{}
			httpRule := &annotations.HttpRule{
				Pattern: &annotations.HttpRule_Get{
					Get: test.uri,
				},
			}
			if err := proto.SetExtension(opts, annotations.E_Http, httpRule); err != nil {
				t.Fatalf("Failed to set google.api.http annotation.")
			}

			// Create a minimal service with a AIP-131 Get method
			// (or with a different method, in the "Irrelevant" case).
			service, err := builder.NewService("Library").AddMethod(builder.NewMethod(test.methodName,
				builder.RpcTypeMessage(builder.NewMessage("GetBookRequest"), false),
				builder.RpcTypeMessage(builder.NewMessage("Book"), false),
			).SetOptions(opts)).Build()
			if err != nil {
				t.Fatalf("Could not build %s method.", test.methodName)
			}

			// Run the method, ensure we get what we expect.
			problems := httpNameField.Lint(service.GetFile())
			if test.msg == "" && len(problems) > 0 {
				t.Errorf("Got %v, expected no problems.", problems)
			} else if test.msg != "" && len(problems) == 0 {
				t.Errorf("Got no problems, expected 1.")
			} else if test.msg != "" && !strings.Contains(problems[0].Message, test.msg) {
				t.Errorf("Got %q, expected message containing %q", problems[0].Message, test.msg)
			}
		})
	}
}
<commit_msg>[refactor] Make AIP-131 tests use testutils.Problems. (#198)<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aip0131

import (
	"strings"
	"testing"

	"github.com/golang/protobuf/proto"
	dpb "github.com/golang/protobuf/protoc-gen-go/descriptor"
	"github.com/googleapis/api-linter/rules/internal/testutils"
	"github.com/jhump/protoreflect/desc/builder"
	"google.golang.org/genproto/googleapis/api/annotations"
)

func TestRequestMessageName(t *testing.T) {
	// Set up the testing permutations.
	tests := []struct {
		testName       string
		methodName     string
		reqMessageName string
		problems       testutils.Problems
	}{
		{"Valid", "GetBook", "GetBookRequest", testutils.Problems{}},
		{"Invalid", "GetBook", "Book", testutils.Problems{{Suggestion: "GetBookRequest"}}},
		{"GetIamPolicy", "GetIamPolicy", "GetIamPolicyRequest", testutils.Problems{}},
		{"Irrelevant", "AcquireBook", "Book", testutils.Problems{}},
	}

	// Run each test individually.
	for _, test := range tests {
		t.Run(test.testName, func(t *testing.T) {
			// Create a minimal service with a AIP-131 Get method
			// (or with a different method, in the "Irrelevant" case).
			service, err := builder.NewService("Library").AddMethod(builder.NewMethod(test.methodName,
				builder.RpcTypeMessage(builder.NewMessage(test.reqMessageName), false),
				builder.RpcTypeMessage(builder.NewMessage("Book"), false),
			)).Build()
			if err != nil {
				t.Fatalf("Could not build %s method.", test.methodName)
			}

			// Run the lint rule, and establish that it returns the expected problems.
			problems := requestMessageName.Lint(service.GetFile())
			if diff := test.problems.SetDescriptor(service.GetMethods()[0]).Diff(problems); diff != "" {
				t.Errorf(diff)
			}
		})
	}
}

func TestResponseMessageName(t *testing.T) {
	// Set up the testing permutations.
	tests := []struct {
		testName        string
		methodName      string
		respMessageName string
		problems        testutils.Problems
	}{
		{"Valid", "GetBook", "Book", testutils.Problems{}},
		{"Invalid", "GetBook", "GetBookResponse", testutils.Problems{{Suggestion: "Book"}}},
		{"Irrelevant", "AcquireBook", "AcquireBookResponse", testutils.Problems{}},
	}

	// Run each test individually.
	for _, test := range tests {
		t.Run(test.testName, func(t *testing.T) {
			// Create a minimal service with a AIP-131 Get method
			// (or with a different method, in the "Irrelevant" case).
			service, err := builder.NewService("Library").AddMethod(builder.NewMethod(test.methodName,
				builder.RpcTypeMessage(builder.NewMessage("GetBookRequest"), false),
				builder.RpcTypeMessage(builder.NewMessage(test.respMessageName), false),
			)).Build()
			if err != nil {
				t.Fatalf("Could not build %s method.", test.methodName)
			}

			// Run the lint rule, and establish that it returns the correct
			// number of problems.
			problems := responseMessageName.Lint(service.GetFile())
			if diff := test.problems.SetDescriptor(service.GetMethods()[0]).Diff(problems); diff != "" {
				t.Errorf(diff)
			}
		})
	}
}

func TestHttpVerb(t *testing.T) {
	// Set up GET and POST HTTP annotations.
	httpGet := &annotations.HttpRule{
		Pattern: &annotations.HttpRule_Get{
			Get: "/v1/{name=publishers/*/books/*}",
		},
	}
	httpPost := &annotations.HttpRule{
		Pattern: &annotations.HttpRule_Post{
			Post: "/v1/{name=publishers/*/books/*}",
		},
	}

	// Set up testing permutations.
	tests := []struct {
		testName   string
		httpRule   *annotations.HttpRule
		methodName string
		msg        string
	}{
		{"Valid", httpGet, "GetBook", ""},
		{"Invalid", httpPost, "GetBook", "HTTP GET"},
		{"Irrelevant", httpPost, "AcquireBook", ""},
	}

	// Run each test.
	for _, test := range tests {
		t.Run(test.testName, func(t *testing.T) {
			// Create a MethodOptions with the annotation set.
			opts := &dpb.MethodOptions{}
			if err := proto.SetExtension(opts, annotations.E_Http, test.httpRule); err != nil {
				t.Fatalf("Failed to set google.api.http annotation.")
			}

			// Create a minimal service with a AIP-131 Get method
			// (or with a different method, in the "Irrelevant" case).
			service, err := builder.NewService("Library").AddMethod(builder.NewMethod(test.methodName,
				builder.RpcTypeMessage(builder.NewMessage("GetBookRequest"), false),
				builder.RpcTypeMessage(builder.NewMessage("Book"), false),
			).SetOptions(opts)).Build()
			if err != nil {
				t.Fatalf("Could not build %s method.", test.methodName)
			}

			// Run the method, ensure we get what we expect.
			problems := httpVerb.Lint(service.GetFile())
			if test.msg == "" && len(problems) > 0 {
				t.Errorf("Got %v, expected no problems.", problems)
			} else if test.msg != "" && !strings.Contains(problems[0].Message, test.msg) {
				t.Errorf("Got %q, expected message containing %q", problems[0].Message, test.msg)
			}
		})
	}
}

func TestHttpBody(t *testing.T) {
	tests := []struct {
		testName   string
		body       string
		methodName string
		msg        string
	}{
		{"Valid", "", "GetBook", ""},
		{"Invalid", "*", "GetBook", "HTTP body"},
		{"Irrelevant", "*", "AcquireBook", ""},
	}

	for _, test := range tests {
		t.Run(test.testName, func(t *testing.T) {
			// Create a MethodOptions with the annotation set.
			opts := &dpb.MethodOptions{}
			httpRule := &annotations.HttpRule{
				Pattern: &annotations.HttpRule_Get{
					Get: "/v1/{name=publishers/*/books/*}",
				},
				Body: test.body,
			}
			if err := proto.SetExtension(opts, annotations.E_Http, httpRule); err != nil {
				t.Fatalf("Failed to set google.api.http annotation.")
			}

			// Create a minimal service with a AIP-131 Get method
			// (or with a different method, in the "Irrelevant" case).
			service, err := builder.NewService("Library").AddMethod(builder.NewMethod(test.methodName,
				builder.RpcTypeMessage(builder.NewMessage("GetBookRequest"), false),
				builder.RpcTypeMessage(builder.NewMessage("Book"), false),
			).SetOptions(opts)).Build()
			if err != nil {
				t.Fatalf("Could not build %s method.", test.methodName)
			}

			// Run the method, ensure we get what we expect.
			problems := httpBody.Lint(service.GetFile())
			if test.msg == "" && len(problems) > 0 {
				t.Errorf("Got %v, expected no problems.", problems)
			} else if test.msg != "" && !strings.Contains(problems[0].Message, test.msg) {
				t.Errorf("Got %q, expected message containing %q", problems[0].Message, test.msg)
			}
		})
	}
}

func TestHttpNameField(t *testing.T) {
	tests := []struct {
		testName   string
		uri        string
		methodName string
		msg        string
	}{
		{"Valid", "/v1/{name=publishers/*/books/*}", "GetBook", ""},
		{"InvalidVarName", "/v1/{book=publishers/*/books/*}", "GetBook", "`name` field"},
		{"NoVarName", "/v1/publishers/*/books/*", "GetBook", "`name` field"},
		{"Irrelevant", "/v1/{book=publishers/*/books/*}", "AcquireBook", ""},
	}

	for _, test := range tests {
		t.Run(test.testName, func(t *testing.T) {
			// Create a MethodOptions with the annotation set.
			opts := &dpb.MethodOptions{}
			httpRule := &annotations.HttpRule{
				Pattern: &annotations.HttpRule_Get{
					Get: test.uri,
				},
			}
			if err := proto.SetExtension(opts, annotations.E_Http, httpRule); err != nil {
				t.Fatalf("Failed to set google.api.http annotation.")
			}

			// Create a minimal service with a AIP-131 Get method
			// (or with a different method, in the "Irrelevant" case).
			service, err := builder.NewService("Library").AddMethod(builder.NewMethod(test.methodName,
				builder.RpcTypeMessage(builder.NewMessage("GetBookRequest"), false),
				builder.RpcTypeMessage(builder.NewMessage("Book"), false),
			).SetOptions(opts)).Build()
			if err != nil {
				t.Fatalf("Could not build %s method.", test.methodName)
			}

			// Run the method, ensure we get what we expect.
			problems := httpNameField.Lint(service.GetFile())
			if test.msg == "" && len(problems) > 0 {
				t.Errorf("Got %v, expected no problems.", problems)
			} else if test.msg != "" && len(problems) == 0 {
				t.Errorf("Got no problems, expected 1.")
			} else if test.msg != "" && !strings.Contains(problems[0].Message, test.msg) {
				t.Errorf("Got %q, expected message containing %q", problems[0].Message, test.msg)
			}
		})
	}
}
<|endoftext|>


<commit_before>// Copyright ©2017 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sampleuv

import (
	"math/rand"
	"sort"
)

// WithoutReplacement samples len(idxs) integers from [0, n) without replacement.
// That is, upon return the elements of idxs will be unique integers. If source
// is non-nil it will be used to generate random numbers, otherwise the default
// source from the math/rand package will be used.
//
// WithoutReplacement will panic if len(idxs) > n.
func WithoutReplacement(idxs []int, n int, src *rand.Rand) {
	if len(idxs) == 0 {
		panic("withoutreplacement: zero length input")
	}
	if len(idxs) > n {
		panic("withoutreplacement: impossible size inputs")
	}

	// There are two algorithms. One is to generate a random permutation
	// and take the first len(idxs) elements. The second is to generate
	// individual random numbers for each element and check uniqueness. The first
	// method scales as O(n), and the second scales as O(len(idxs)^2). Choose
	// the algorithm accordingly.
	if n < len(idxs)*len(idxs) {
		var perm []int
		if src != nil {
			perm = src.Perm(n)
		} else {
			perm = rand.Perm(n)
		}
		copy(idxs, perm)
	}

	// Instead, generate the random numbers directly.
	sorted := make([]int, 0, len(idxs))
	for i := range idxs {
		var r int
		if src != nil {
			r = src.Intn(n - i)
		} else {
			r = rand.Intn(n - i)
		}
		for _, v := range sorted {
			if r >= v {
				r++
			}
		}
		idxs[i] = r
		sorted = append(sorted, r)
		sort.Ints(sorted)
	}
}
<commit_msg>stat/sampleuv: add missing return<commit_after>// Copyright ©2017 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sampleuv

import (
	"math/rand"
	"sort"
)

// WithoutReplacement samples len(idxs) integers from [0, n) without replacement.
// That is, upon return the elements of idxs will be unique integers. If source
// is non-nil it will be used to generate random numbers, otherwise the default
// source from the math/rand package will be used.
//
// WithoutReplacement will panic if len(idxs) > n.
func WithoutReplacement(idxs []int, n int, src *rand.Rand) {
	if len(idxs) == 0 {
		panic("withoutreplacement: zero length input")
	}
	if len(idxs) > n {
		panic("withoutreplacement: impossible size inputs")
	}

	// There are two algorithms. One is to generate a random permutation
	// and take the first len(idxs) elements. The second is to generate
	// individual random numbers for each element and check uniqueness. The first
	// method scales as O(n), and the second scales as O(len(idxs)^2). Choose
	// the algorithm accordingly.
	if n < len(idxs)*len(idxs) {
		var perm []int
		if src != nil {
			perm = src.Perm(n)
		} else {
			perm = rand.Perm(n)
		}
		copy(idxs, perm)
		return
	}

	// Instead, generate the random numbers directly.
	sorted := make([]int, 0, len(idxs))
	for i := range idxs {
		var r int
		if src != nil {
			r = src.Intn(n - i)
		} else {
			r = rand.Intn(n - i)
		}
		for _, v := range sorted {
			if r >= v {
				r++
			}
		}
		idxs[i] = r
		sorted = append(sorted, r)
		sort.Ints(sorted)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"reflect"
	"testing"
)

func Test_parseCGIHeaders(t *testing.T) {
	data := []struct {
		in      string
		out     string
		headers map[string]string
	}{
		{
			in:      "Some text",
			out:     "Some text",
			headers: map[string]string{},
		},
		{
			in:      "Location: url\n\nSome text",
			out:     "Some text",
			headers: map[string]string{"Location": "url"},
		},
		{
			in:      "Location: url\n\n",
			out:     "",
			headers: map[string]string{"Location": "url"},
		},
		{
			in:      "Location: url\nX-Name:  x-value\n\nSome text",
			out:     "Some text",
			headers: map[string]string{"Location": "url", "X-Name": "x-value"},
		},
	}

	for i, item := range data {
		out, headers := parseCGIHeaders(item.in)
		if !reflect.DeepEqual(item.headers, headers) || item.out != out {
			t.Errorf("%d:\nexpected: %s / %#v\nreal    : %s / %#v", i, item.out, item.headers, out, headers)
		}
	}
}

func Test_getShellAndParams(t *testing.T) {
	shell, params, err := getShellAndParams("ls", "sh", false)
	if shell != "sh" || !reflect.DeepEqual(params, []string{"-c", "ls"}) || err != nil {
		t.Errorf("1. getShellAndParams() failed")
	}

	shell, params, err = getShellAndParams("ls", "sh", true)
	if shell != "cmd" || !reflect.DeepEqual(params, []string{"/C", "ls"}) || err != nil {
		t.Errorf("2. getShellAndParams() failed")
	}

	shell, params, err = getShellAndParams("ls", "bash", false)
	if shell != "bash" || !reflect.DeepEqual(params, []string{"-c", "ls"}) || err != nil {
		t.Errorf("3. getShellAndParams() failed")
	}

	shell, params, err = getShellAndParams("ls -l -a", "", false)
	if shell != "ls" || !reflect.DeepEqual(params, []string{"-l", "-a"}) || err != nil {
		t.Errorf("4. getShellAndParams() failed")
	}

	shell, params, err = getShellAndParams("ls -l 'a b'", "", false)
	if shell != "ls" || !reflect.DeepEqual(params, []string{"-l", "a b"}) || err != nil {
		t.Errorf("5. getShellAndParams() failed")
	}

	shell, params, err = getShellAndParams("ls '-l", "", false)
	if err == nil {
		t.Errorf("6. getShellAndParams() failed")
	}
}
<commit_msg>First version of test main<commit_after>package main

import (
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"reflect"
	"strings"
	"testing"
)

func Test_parseCGIHeaders(t *testing.T) {
	data := []struct {
		in      string
		out     string
		headers map[string]string
	}{
		{
			in:      "Some text",
			out:     "Some text",
			headers: map[string]string{},
		},
		{
			in:      "Location: url\n\nSome text",
			out:     "Some text",
			headers: map[string]string{"Location": "url"},
		},
		{
			in:      "Location: url\n\n",
			out:     "",
			headers: map[string]string{"Location": "url"},
		},
		{
			in:      "Location: url\nX-Name:  x-value\n\nSome text",
			out:     "Some text",
			headers: map[string]string{"Location": "url", "X-Name": "x-value"},
		},
	}

	for i, item := range data {
		out, headers := parseCGIHeaders(item.in)
		if !reflect.DeepEqual(item.headers, headers) || item.out != out {
			t.Errorf("%d:\nexpected: %s / %#v\nreal    : %s / %#v", i, item.out, item.headers, out, headers)
		}
	}
}

func Test_getShellAndParams(t *testing.T) {
	shell, params, err := getShellAndParams("ls", "sh", false)
	if shell != "sh" || !reflect.DeepEqual(params, []string{"-c", "ls"}) || err != nil {
		t.Errorf("1. getShellAndParams() failed")
	}

	shell, params, err = getShellAndParams("ls", "sh", true)
	if shell != "cmd" || !reflect.DeepEqual(params, []string{"/C", "ls"}) || err != nil {
		t.Errorf("2. getShellAndParams() failed")
	}

	shell, params, err = getShellAndParams("ls", "bash", false)
	if shell != "bash" || !reflect.DeepEqual(params, []string{"-c", "ls"}) || err != nil {
		t.Errorf("3. getShellAndParams() failed")
	}

	shell, params, err = getShellAndParams("ls -l -a", "", false)
	if shell != "ls" || !reflect.DeepEqual(params, []string{"-l", "-a"}) || err != nil {
		t.Errorf("4. getShellAndParams() failed")
	}

	shell, params, err = getShellAndParams("ls -l 'a b'", "", false)
	if shell != "ls" || !reflect.DeepEqual(params, []string{"-l", "a b"}) || err != nil {
		t.Errorf("5. getShellAndParams() failed")
	}

	shell, params, err = getShellAndParams("ls '-l", "", false)
	if err == nil {
		t.Errorf("6. getShellAndParams() failed")
	}
}

func httpGet(url string) ([]byte, error) {
	res, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	body, err := ioutil.ReadAll(res.Body)
	defer res.Body.Close()
	if err != nil {
		return nil, err
	}

	return body, nil
}

func getFreePort() string {
	listen, _ := net.Listen("tcp", ":0")
	defer listen.Close()
	parts := strings.Split(listen.Addr().String(), ":")

	return parts[len(parts)-1]
}

func Test_main1(t *testing.T) {
	port := getFreePort()
	os.Args = []string{"shell2http",
		"-add-exit",
		"-cache=1",
		"-cgi",
		"-export-all-vars",
		"-form",
		"-one-thread",
		"-shell=bash",
		"-log=/dev/null",
		"-port=" + port,
		"/echo", "echo 123"}
	go main()

	res, err := httpGet("http://localhost:" + port + "/")
	if err != nil {
		t.Errorf("1. main() failed: %s", err)
	}
	if len(res) == 0 || !strings.HasPrefix(string(res), "<!DOCTYPE html>") {
		t.Errorf("1. main() failed: real result: '%s'", string(res))
	}

	res, err = httpGet("http://localhost:" + port + "/echo")
	if err != nil {
		t.Errorf("2. main() failed: %s", err)
	}
	if string(res) != "123\n" {
		t.Errorf("2. main() failed: real result: '%s'", string(res))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/google/go-cmp/cmp/cmpopts"
	"github.com/influxdata/flux"
	"github.com/influxdata/flux/semantic/semantictest"
	platform "github.com/influxdata/influxdb/v2"
	"github.com/influxdata/influxdb/v2/mock"
	"github.com/influxdata/influxdb/v2/query/influxql"
	platformtesting "github.com/influxdata/influxdb/v2/testing"
)

func printUsage() {
	fmt.Println("usage: prepcsvtests /path/to/testfiles [testname]")
}

func main() {
	fnames := make([]string, 0)
	path := ""
	var err error
	if len(os.Args) == 3 {
		path = os.Args[1]
		fnames = append(fnames, filepath.Join(path, os.Args[2])+".flux")
	} else if len(os.Args) == 2 {
		path = os.Args[1]
		fnames, err = filepath.Glob(filepath.Join(path, "*.flux"))
		if err != nil {
			return
		}
	} else {
		printUsage()
		return
	}

	for _, fname := range fnames {
		ext := ".flux"
		testName := fname[0 : len(fname)-len(ext)]

		fluxText, err := ioutil.ReadFile(fname)
		if err != nil {
			fmt.Printf("error reading ifq	l query text: %s", err)
			return
		}

		influxqlText, err := ioutil.ReadFile(testName + ".influxql")
		if err != nil {
			fmt.Printf("error reading influxql query text: %s", err)
			return
		}

		fluxSpec, err := flux.Compile(context.Background(), string(fluxText), time.Now().UTC())
		if err != nil {
			fmt.Printf("error compiling. \n query: \n %s \n err: %s", string(fluxText), err)
			return
		}

		transpiler := influxql.NewTranspiler(dbrpMappingSvc)
		influxqlSpec, err := transpiler.Transpile(context.Background(), string(influxqlText))
		if err != nil {
			fmt.Printf("error transpiling. \n query: \n %s \n err: %s", string(influxqlText), err)
			return
		}
		var opts = append(
			semantictest.CmpOptions,
			cmp.AllowUnexported(flux.Spec{}),
			cmpopts.IgnoreUnexported(flux.Spec{}))

		difference := cmp.Diff(fluxSpec, influxqlSpec, opts...)

		fmt.Printf("compiled vs transpiled diff: \n%s", difference)
	}
}

// Setup mock DBRPMappingService to always return `db.rp`.
var dbrpMappingSvc = mock.NewDBRPMappingService()

func init() {
	mapping := platform.DBRPMapping{
		Cluster:         "cluster",
		Database:        "db",
		RetentionPolicy: "rp",
		Default:         true,
		OrganizationID:  platformtesting.MustIDBase16("aaaaaaaaaaaaaaaa"),
		BucketID:        platformtesting.MustIDBase16("bbbbbbbbbbbbbbbb"),
	}
	dbrpMappingSvc.FindByFn = func(ctx context.Context, cluster string, db string, rp string) (*platform.DBRPMapping, error) {
		return &mapping, nil
	}
	dbrpMappingSvc.FindFn = func(ctx context.Context, filter platform.DBRPMappingFilter) (*platform.DBRPMapping, error) {
		return &mapping, nil
	}
	dbrpMappingSvc.FindManyFn = func(ctx context.Context, filter platform.DBRPMappingFilter, opt ...platform.FindOptions) ([]*platform.DBRPMapping, int, error) {
		return []*platform.DBRPMapping{&mapping}, 1, nil
	}
}
<commit_msg>chore: delete unused, broken 'compspecs' testing tool (#22335)<commit_after><|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
)

type SingleRule struct {
	color string
	regex string
}

type MultiRule struct {
	color string
	start string
	end   string
}

// JoinRule takes a syntax rule (which can be multiple regular expressions)
// and joins it into one regular expression by ORing everything together
func JoinRule(rule string) string {
	split := strings.Split(rule, `" "`)
	joined := strings.Join(split, "|")
	joined = joined
	return joined
}

func parseFile(text, filename string) (filetype, syntax, header string, rules []interface{}) {
	lines := strings.Split(text, "\n")

	// Regex for parsing syntax statements
	syntaxParser := regexp.MustCompile(`syntax "(.*?)"\s+"(.*)"+`)
	// Regex for parsing header statements
	headerParser := regexp.MustCompile(`header "(.*)"`)

	// Regex for parsing standard syntax rules
	ruleParser := regexp.MustCompile(`color (.*?)\s+(?:\((.+?)?\)\s+)?"(.*)"`)
	// Regex for parsing syntax rules with start="..." end="..."
	ruleStartEndParser := regexp.MustCompile(`color (.*?)\s+(?:\((.+?)?\)\s+)?start="(.*)"\s+end="(.*)"`)

	for lineNum, line := range lines {
		line = strings.TrimSpace(line)
		if line == "" {
			continue
		}
		if strings.HasPrefix(line, "#") {
			continue
		}
		if strings.HasPrefix(line, "syntax") {
			syntaxMatches := syntaxParser.FindSubmatch([]byte(line))
			if len(syntaxMatches) == 3 {
				filetype = string(syntaxMatches[1])
				syntax = JoinRule(string(syntaxMatches[2]))
			} else {
				fmt.Println(filename, lineNum, "Syntax statement is not valid: "+line)
				continue
			}
		}
		if strings.HasPrefix(line, "header") {
			// Header statement
			headerMatches := headerParser.FindSubmatch([]byte(line))
			if len(headerMatches) == 2 {
				header = JoinRule(string(headerMatches[1]))
			} else {
				fmt.Println(filename, lineNum, "Header statement is not valid: "+line)
				continue
			}
		}

		// Syntax rule, but it could be standard or start-end
		if ruleParser.MatchString(line) {
			// Standard syntax rule
			// Parse the line
			submatch := ruleParser.FindSubmatch([]byte(line))
			var color string
			var regexStr string
			var flags string
			if len(submatch) == 4 {
				// If len is 4 then the user specified some additional flags to use
				color = string(submatch[1])
				flags = string(submatch[2])
				if flags != "" {
					regexStr = "(?" + flags + ")" + JoinRule(string(submatch[3]))
				} else {
					regexStr = JoinRule(string(submatch[3]))
				}
			} else if len(submatch) == 3 {
				// If len is 3, no additional flags were given
				color = string(submatch[1])
				regexStr = JoinRule(string(submatch[2]))
			} else {
				// If len is not 3 or 4 there is a problem
				fmt.Println(filename, lineNum, "Invalid statement: "+line)
				continue
			}

			rules = append(rules, SingleRule{color, regexStr})
		} else if ruleStartEndParser.MatchString(line) {
			// Start-end syntax rule
			submatch := ruleStartEndParser.FindSubmatch([]byte(line))
			var color string
			var start string
			var end string
			// Use m and s flags by default
			flags := "ms"
			if len(submatch) == 5 {
				// If len is 5 the user provided some additional flags
				color = string(submatch[1])
				flags += string(submatch[2])
				start = string(submatch[3])
				end = string(submatch[4])
			} else if len(submatch) == 4 {
				// If len is 4 the user did not provide additional flags
				color = string(submatch[1])
				start = string(submatch[2])
				end = string(submatch[3])
			} else {
				// If len is not 4 or 5 there is a problem
				fmt.Println(filename, lineNum, "Invalid statement: "+line)
				continue
			}

			// rules[color] = "(?" + flags + ")" + "(" + start + ").*?(" + end + ")"
			rules = append(rules, MultiRule{color, start, end})
		}
	}

	return
}

func generateFile(filetype, syntax, header string, rules []interface{}) string {
	output := ""

	output += fmt.Sprintf("filetype: %s\n\n", filetype)
	output += fmt.Sprintf("detect: \n\tfilename: \"%s\"\n", strings.Replace(syntax, "\\", "\\\\", -1))

	if header != "" {
		output += fmt.Sprintf("\theader: \"%s\"\n", strings.Replace(header, "\\", "\\\\", -1))
	}

	output += "\nrules:\n"

	for _, r := range rules {
		if rule, ok := r.(SingleRule); ok {
			output += fmt.Sprintf("\t- %s: \"%s\"\n", rule.color, strings.Replace(strings.Replace(rule.regex, "\\", "\\\\", -1), "\"", "\\\"", -1))
		} else if rule, ok := r.(MultiRule); ok {
			output += fmt.Sprintf("\t- %s:\n", rule.color)
			output += fmt.Sprintf("\t\tstart: \"%s\"\n", rule.start)
			output += fmt.Sprintf("\t\tend: \"%s\"\n", rule.end)
			output += fmt.Sprintf("\t\trules: []\n\n")
		}
	}

	return output
}

func main() {
	if len(os.Args) < 2 {
		fmt.Println("no args")
		return
	}

	data, _ := ioutil.ReadFile(os.Args[1])
	fmt.Print(generateFile(parseFile(string(data), os.Args[1])))
}
<commit_msg>No tabs in yaml<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
)

type SingleRule struct {
	color string
	regex string
}

type MultiRule struct {
	color string
	start string
	end   string
}

// JoinRule takes a syntax rule (which can be multiple regular expressions)
// and joins it into one regular expression by ORing everything together
func JoinRule(rule string) string {
	split := strings.Split(rule, `" "`)
	joined := strings.Join(split, "|")
	joined = joined
	return joined
}

func parseFile(text, filename string) (filetype, syntax, header string, rules []interface{}) {
	lines := strings.Split(text, "\n")

	// Regex for parsing syntax statements
	syntaxParser := regexp.MustCompile(`syntax "(.*?)"\s+"(.*)"+`)
	// Regex for parsing header statements
	headerParser := regexp.MustCompile(`header "(.*)"`)

	// Regex for parsing standard syntax rules
	ruleParser := regexp.MustCompile(`color (.*?)\s+(?:\((.+?)?\)\s+)?"(.*)"`)
	// Regex for parsing syntax rules with start="..." end="..."
	ruleStartEndParser := regexp.MustCompile(`color (.*?)\s+(?:\((.+?)?\)\s+)?start="(.*)"\s+end="(.*)"`)

	for lineNum, line := range lines {
		line = strings.TrimSpace(line)
		if line == "" {
			continue
		}
		if strings.HasPrefix(line, "#") {
			continue
		}
		if strings.HasPrefix(line, "syntax") {
			syntaxMatches := syntaxParser.FindSubmatch([]byte(line))
			if len(syntaxMatches) == 3 {
				filetype = string(syntaxMatches[1])
				syntax = JoinRule(string(syntaxMatches[2]))
			} else {
				fmt.Println(filename, lineNum, "Syntax statement is not valid: "+line)
				continue
			}
		}
		if strings.HasPrefix(line, "header") {
			// Header statement
			headerMatches := headerParser.FindSubmatch([]byte(line))
			if len(headerMatches) == 2 {
				header = JoinRule(string(headerMatches[1]))
			} else {
				fmt.Println(filename, lineNum, "Header statement is not valid: "+line)
				continue
			}
		}

		// Syntax rule, but it could be standard or start-end
		if ruleParser.MatchString(line) {
			// Standard syntax rule
			// Parse the line
			submatch := ruleParser.FindSubmatch([]byte(line))
			var color string
			var regexStr string
			var flags string
			if len(submatch) == 4 {
				// If len is 4 then the user specified some additional flags to use
				color = string(submatch[1])
				flags = string(submatch[2])
				if flags != "" {
					regexStr = "(?" + flags + ")" + JoinRule(string(submatch[3]))
				} else {
					regexStr = JoinRule(string(submatch[3]))
				}
			} else if len(submatch) == 3 {
				// If len is 3, no additional flags were given
				color = string(submatch[1])
				regexStr = JoinRule(string(submatch[2]))
			} else {
				// If len is not 3 or 4 there is a problem
				fmt.Println(filename, lineNum, "Invalid statement: "+line)
				continue
			}

			rules = append(rules, SingleRule{color, regexStr})
		} else if ruleStartEndParser.MatchString(line) {
			// Start-end syntax rule
			submatch := ruleStartEndParser.FindSubmatch([]byte(line))
			var color string
			var start string
			var end string
			// Use m and s flags by default
			flags := "ms"
			if len(submatch) == 5 {
				// If len is 5 the user provided some additional flags
				color = string(submatch[1])
				flags += string(submatch[2])
				start = string(submatch[3])
				end = string(submatch[4])
			} else if len(submatch) == 4 {
				// If len is 4 the user did not provide additional flags
				color = string(submatch[1])
				start = string(submatch[2])
				end = string(submatch[3])
			} else {
				// If len is not 4 or 5 there is a problem
				fmt.Println(filename, lineNum, "Invalid statement: "+line)
				continue
			}

			// rules[color] = "(?" + flags + ")" + "(" + start + ").*?(" + end + ")"
			rules = append(rules, MultiRule{color, start, end})
		}
	}

	return
}

func generateFile(filetype, syntax, header string, rules []interface{}) string {
	output := ""

	output += fmt.Sprintf("filetype: %s\n\n", filetype)
	output += fmt.Sprintf("detect: \n    filename: \"%s\"\n", strings.Replace(syntax, "\\", "\\\\", -1))

	if header != "" {
		output += fmt.Sprintf("    header: \"%s\"\n", strings.Replace(header, "\\", "\\\\", -1))
	}

	output += "\nrules:\n"

	for _, r := range rules {
		if rule, ok := r.(SingleRule); ok {
			output += fmt.Sprintf("    - %s: \"%s\"\n", rule.color, strings.Replace(strings.Replace(rule.regex, "\\", "\\\\", -1), "\"", "\\\"", -1))
		} else if rule, ok := r.(MultiRule); ok {
			output += fmt.Sprintf("    - %s:\n", rule.color)
			output += fmt.Sprintf("        start: \"%s\"\n", rule.start)
			output += fmt.Sprintf("        end: \"%s\"\n", rule.end)
			output += fmt.Sprintf("        rules: []\n\n")
		}
	}

	return output
}

func main() {
	if len(os.Args) < 2 {
		fmt.Println("no args")
		return
	}

	data, _ := ioutil.ReadFile(os.Args[1])
	fmt.Print(generateFile(parseFile(string(data), os.Args[1])))
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"socialapi/workers/common/handler"
	"socialapi/workers/common/response"
	"socialapi/workers/realtime/models"
	"time"

	"github.com/koding/logging"
)

type Handler struct {
	pubnub *models.Pubnub
	logger logging.Logger
}

func NewHandler(p *models.Pubnub, l logging.Logger) *Handler {
	return &Handler{
		pubnub: p,
		logger: l,
	}
}

// SubscribeChannel checks users channel accessability and regarding to that
// grants channel access for them
func (h *Handler) SubscribeChannel(u *url.URL, header http.Header, req *models.Channel) (int, http.Header, interface{}, error) {
	res, err := checkParticipation(u, header, req)
	if err != nil {
		return response.NewAccessDenied(err)
	}

	// user has access permission, now authenticate user to channel via pubnub
	a := new(models.Authenticate)
	a.Channel = models.NewPrivateMessageChannel(*res.Channel)
	a.Account = res.Account

	err = h.pubnub.Authenticate(a)
	if err != nil {
		return response.NewBadRequest(err)
	}

	return responseWithCookie(req, a.Account.Token)
}

// SubscribeNotification grants notification channel access for user. User information is
// fetched from session
func (h *Handler) SubscribeNotification(u *url.URL, header http.Header, temp *models.Account) (int, http.Header, interface{}, error) {

	// fetch account information from session
	account, err := getAccountInfo(u, header)
	if err != nil {
		return response.NewBadRequest(err)
	}

	// authenticate user to their notification channel
	a := new(models.Authenticate)
	a.Channel = models.NewNotificationChannel(account)
	a.Account = account

	err = h.pubnub.Authenticate(a)
	if err != nil {
		return response.NewBadRequest(err)
	}

	return responseWithCookie(temp, account.Token)
}

func (h *Handler) SubscribeMessage(u *url.URL, header http.Header, um *models.UpdateInstanceMessage) (int, http.Header, interface{}, error) {
	if um.Token == "" {
		return response.NewBadRequest(models.ErrTokenNotSet)
	}

	a := new(models.Authenticate)
	a.Channel = models.NewMessageUpdateChannel(*um)
	err := h.pubnub.Authenticate(a)
	if err != nil {
		return response.NewBadRequest(err)
	}

	return response.NewOK(um)
}

func responseWithCookie(req interface{}, token string) (int, http.Header, interface{}, error) {
	expires := time.Now().AddDate(5, 0, 0)
	cookie := &http.Cookie{
		Name:       "realtimeToken",
		Value:      token,
		Path:       "/",
		Domain:     "lvh.me", // TODO change this
		Expires:    expires,
		RawExpires: expires.Format(time.UnixDate),
		Raw:        "realtimeToken=" + token,
		Unparsed:   []string{"realtimeToken=" + token},
	}

	return response.NewOKWithCookie(req, []*http.Cookie{cookie})
}

// TODO needs a better request handler
func checkParticipation(u *url.URL, header http.Header, cr *models.Channel) (*models.CheckParticipationResponse, error) {
	// relay the cookie to other endpoint
	cookie := header.Get("Cookie")
	request := &handler.Request{
		Type:     "GET",
		Endpoint: "/api/social/channel/checkparticipation",
		Params: map[string]string{
			"name":  cr.Name,
			"group": cr.Group,
			"type":  cr.Type,
		},
		Cookie: cookie,
	}

	// TODO update this requester
	resp, err := handler.MakeRequest(request)
	if err != nil {
		return nil, err
	}

	// Need a better response
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf(resp.Status)
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var cpr models.CheckParticipationResponse
	err = json.Unmarshal(body, &cpr)
	if err != nil {
		return nil, err
	}

	return &cpr, nil
}

func getAccountInfo(u *url.URL, header http.Header) (*models.Account, error) {
	cookie := header.Get("Cookie")
	request := &handler.Request{
		Type:     "GET",
		Endpoint: "/api/social/account",
		Cookie:   cookie,
	}

	// TODO update this requester
	resp, err := handler.MakeRequest(request)
	if err != nil {
		return nil, err
	}

	// Need a better response
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf(resp.Status)
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var a models.Account
	err = json.Unmarshal(body, &a)
	if err != nil {
		return nil, err
	}

	return &a, nil
}
<commit_msg>gatekeeper: remove domain setter in cookie<commit_after>package api

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"socialapi/workers/common/handler"
	"socialapi/workers/common/response"
	"socialapi/workers/realtime/models"
	"time"

	"github.com/koding/logging"
)

type Handler struct {
	pubnub *models.Pubnub
	logger logging.Logger
}

func NewHandler(p *models.Pubnub, l logging.Logger) *Handler {
	return &Handler{
		pubnub: p,
		logger: l,
	}
}

// SubscribeChannel checks users channel accessability and regarding to that
// grants channel access for them
func (h *Handler) SubscribeChannel(u *url.URL, header http.Header, req *models.Channel) (int, http.Header, interface{}, error) {
	res, err := checkParticipation(u, header, req)
	if err != nil {
		return response.NewAccessDenied(err)
	}

	// user has access permission, now authenticate user to channel via pubnub
	a := new(models.Authenticate)
	a.Channel = models.NewPrivateMessageChannel(*res.Channel)
	a.Account = res.Account

	err = h.pubnub.Authenticate(a)
	if err != nil {
		return response.NewBadRequest(err)
	}

	return responseWithCookie(req, a.Account.Token)
}

// SubscribeNotification grants notification channel access for user. User information is
// fetched from session
func (h *Handler) SubscribeNotification(u *url.URL, header http.Header, temp *models.Account) (int, http.Header, interface{}, error) {

	// fetch account information from session
	account, err := getAccountInfo(u, header)
	if err != nil {
		return response.NewBadRequest(err)
	}

	// authenticate user to their notification channel
	a := new(models.Authenticate)
	a.Channel = models.NewNotificationChannel(account)
	a.Account = account

	err = h.pubnub.Authenticate(a)
	if err != nil {
		return response.NewBadRequest(err)
	}

	return responseWithCookie(temp, account.Token)
}

func (h *Handler) SubscribeMessage(u *url.URL, header http.Header, um *models.UpdateInstanceMessage) (int, http.Header, interface{}, error) {
	if um.Token == "" {
		return response.NewBadRequest(models.ErrTokenNotSet)
	}

	a := new(models.Authenticate)
	a.Channel = models.NewMessageUpdateChannel(*um)
	err := h.pubnub.Authenticate(a)
	if err != nil {
		return response.NewBadRequest(err)
	}

	return response.NewOK(um)
}

func responseWithCookie(req interface{}, token string) (int, http.Header, interface{}, error) {
	expires := time.Now().AddDate(5, 0, 0)
	cookie := &http.Cookie{
		Name:       "realtimeToken",
		Value:      token,
		Path:       "/",
		Expires:    expires,
		RawExpires: expires.Format(time.UnixDate),
		Raw:        "realtimeToken=" + token,
		Unparsed:   []string{"realtimeToken=" + token},
	}

	return response.NewOKWithCookie(req, []*http.Cookie{cookie})
}

// TODO needs a better request handler
func checkParticipation(u *url.URL, header http.Header, cr *models.Channel) (*models.CheckParticipationResponse, error) {
	// relay the cookie to other endpoint
	cookie := header.Get("Cookie")
	request := &handler.Request{
		Type:     "GET",
		Endpoint: "/api/social/channel/checkparticipation",
		Params: map[string]string{
			"name":  cr.Name,
			"group": cr.Group,
			"type":  cr.Type,
		},
		Cookie: cookie,
	}

	// TODO update this requester
	resp, err := handler.MakeRequest(request)
	if err != nil {
		return nil, err
	}

	// Need a better response
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf(resp.Status)
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var cpr models.CheckParticipationResponse
	err = json.Unmarshal(body, &cpr)
	if err != nil {
		return nil, err
	}

	return &cpr, nil
}

func getAccountInfo(u *url.URL, header http.Header) (*models.Account, error) {
	cookie := header.Get("Cookie")
	request := &handler.Request{
		Type:     "GET",
		Endpoint: "/api/social/account",
		Cookie:   cookie,
	}

	// TODO update this requester
	resp, err := handler.MakeRequest(request)
	if err != nil {
		return nil, err
	}

	// Need a better response
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf(resp.Status)
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var a models.Account
	err = json.Unmarshal(body, &a)
	if err != nil {
		return nil, err
	}

	return &a, nil
}
<|endoftext|>


<commit_before>package rpm

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"math"
)

// A Header stores metadata about a rpm package.
type Header struct {
	Version    int
	IndexCount int
	Length     int
	Indexes    IndexEntries
	Start      int
	End        int
}

// Headers is an array of Header structs.
type Headers []Header

// Predefined sizing constraints.
const (
	// MAX_HEADER_SIZE is the maximum allowable header size in bytes (32 MB).
	MAX_HEADER_SIZE = 33554432
)

// Predefined header errors.
var (
	// ErrBadHeaderLength indicates that the read header section is not the
	// expected length.
	ErrBadHeaderLength = fmt.Errorf("RPM header section is incorrect length")

	// ErrNotHeader indicates that the read header section does start with the
	// expected descriptor.
	ErrNotHeader = fmt.Errorf("invalid RPM header descriptor")

	// ErrBadStoreLength indicates that the read header store section is not the
	// expected length.
	ErrBadStoreLength = fmt.Errorf("header value store is incorrect length")
)

// Predefined header index errors.
var (
	// ErrBadIndexCount indicates that number of indexes given in the read
	// header would exceed the actual size of the header.
	ErrBadIndexCount = fmt.Errorf("index count exceeds header size")

	// ErrBadIndexLength indicates that the read header index section is not the
	// expected length.
	ErrBadIndexLength = fmt.Errorf("index section is incorrect length")

	// ErrIndexOutOfRange indicates that the read header index would exceed the
	// range of the header.
	ErrIndexOutOfRange = fmt.Errorf("index is out of range")

	// ErrBadIndexType indicates that the read index contains a value of an
	// unsupported data type.
	ErrBadIndexType = fmt.Errorf("unknown index data type")

	// ErrBadIndexValueCount indicates that the read index value would exceed
	// the range of the header store section.
	ErrBadIndexValueCount = fmt.Errorf("index value count is out of range")
)

// ReadPackageHeader reads an RPM package file header structure from the given
// io.Reader.
//
// This function should only be used if you intend to read a package header
// structure in isolation.
func ReadPackageHeader(r io.Reader) (*Header, error) {
	// read the "header structure header"
	header := make([]byte, 16)
	n, err := r.Read(header)
	if err != nil {
		return nil, err
	}

	if n != 16 {
		return nil, ErrBadHeaderLength
	}

	// check magic number
	if 0 != bytes.Compare(header[:3], []byte{0x8E, 0xAD, 0xE8}) {
		return nil, ErrNotHeader
	}

	// translate header
	h := &Header{
		Version:    int(header[3]),
		IndexCount: int(binary.BigEndian.Uint32(header[8:12])),
		Length:     int(binary.BigEndian.Uint32(header[12:16])),
	}

	// make sure header size is in range
	if h.Length > MAX_HEADER_SIZE {
		return nil, ErrBadHeaderLength
	}

	// Ensure index count is in range
	// This test is not entirely precise as h.Length also includes the value
	// store. It should at least help eliminate excessive buffer allocations for
	// corrupted length values in the > h.Length ranges.
	if h.IndexCount*16 > h.Length {
		return nil, ErrBadIndexCount
	}

	h.Indexes = make(IndexEntries, h.IndexCount)

	// read indexes
	indexLength := 16 * h.IndexCount
	indexes := make([]byte, indexLength)
	n, err = r.Read(indexes)
	if err != nil {
		return nil, err
	}

	if n != indexLength {
		return nil, ErrBadIndexLength
	}

	for x := 0; x < h.IndexCount; x++ {
		o := 16 * x
		index := IndexEntry{
			Tag:       int(binary.BigEndian.Uint32(indexes[o : o+4])),
			Type:      int(binary.BigEndian.Uint32(indexes[o+4 : o+8])),
			Offset:    int(binary.BigEndian.Uint32(indexes[o+8 : o+12])),
			ItemCount: int(binary.BigEndian.Uint32(indexes[o+12 : o+16])),
		}

		// validate index offset
		if index.Offset >= h.Length {
			return nil, ErrIndexOutOfRange
		}

		// append
		h.Indexes[x] = index
	}

	// read the "store"
	store := make([]byte, h.Length)
	n, err = r.Read(store)
	if err != nil {
		return nil, err
	}

	if n != h.Length {
		return nil, ErrBadStoreLength
	}

	// parse the value of each index from the store
	for x := 0; x < h.IndexCount; x++ {
		index := h.Indexes[x]
		o := index.Offset

		if index.ItemCount == 0 {
			return nil, ErrBadIndexValueCount
		}

		switch index.Type {
		case IndexDataTypeChar:
			vals := make([]uint8, index.ItemCount)
			for v := 0; v < index.ItemCount; v++ {
				if o >= len(store) {
					return nil, fmt.Errorf("uint8 value for index %d is out of range", x+1)
				}

				vals[v] = uint8(store[o])
				o += 1
			}

			index.Value = vals

		case IndexDataTypeInt8:
			vals := make([]int8, index.ItemCount)
			for v := 0; v < index.ItemCount; v++ {
				if o >= len(store) {
					return nil, fmt.Errorf("int8 value for index %d is out of range", x+1)
				}

				vals[v] = int8(store[o])
				o += 1
			}

			index.Value = vals

		case IndexDataTypeInt16:
			vals := make([]int16, index.ItemCount)
			for v := 0; v < index.ItemCount; v++ {
				if o+2 > len(store) {
					return nil, fmt.Errorf("int16 value for index %d is out of range", x+1)
				}

				vals[v] = int16(binary.BigEndian.Uint16(store[o : o+2]))
				o += 2
			}

			index.Value = vals

		case IndexDataTypeInt32:
			vals := make([]int32, index.ItemCount)
			for v := 0; v < index.ItemCount; v++ {
				if o+4 > len(store) {
					return nil, fmt.Errorf("int32 value for index %d is out of range", x+1)
				}

				vals[v] = int32(binary.BigEndian.Uint32(store[o : o+4]))
				o += 4
			}

			index.Value = vals

		case IndexDataTypeInt64:
			vals := make([]int64, index.ItemCount)
			for v := 0; v < index.ItemCount; v++ {
				if o+8 > len(store) {
					return nil, fmt.Errorf("int64 value for index %d is out of range", x+1)
				}

				vals[v] = int64(binary.BigEndian.Uint64(store[o : o+8]))
				o += 8
			}

			index.Value = vals

		case IndexDataTypeBinary:
			if o+index.ItemCount > len(store) {
				return nil, fmt.Errorf("[]byte value for index %d is out of range", x+1)
			}

			b := make([]byte, index.ItemCount)
			copy(b, store[o:o+index.ItemCount])

			index.Value = b

		case IndexDataTypeString, IndexDataTypeStringArray, IndexDataTypeI8NString:
			// allow atleast one byte per string
			if o+index.ItemCount > len(store) {
				return nil, fmt.Errorf("[]string value for index %d is out of range", x+1)
			}

			vals := make([]string, index.ItemCount)

			for s := 0; s < index.ItemCount; s++ {
				// calculate string length
				var j int
				for j = 0; (o+j) < len(store) && store[o+j] != 0; j++ {
				}

				if j == len(store) {
					return nil, fmt.Errorf("string value for index %d is out of range", x+1)
				}

				vals[s] = string(store[o : o+j])
				o += j + 1
			}

			index.Value = vals

		case IndexDataTypeNull:
			// nothing to do here

		default:
			// unknown data type
			return nil, ErrBadIndexType
		}

		// save in array
		h.Indexes[x] = index
	}

	// calculate location of the end of the header by padding to a multiple of 8
	o := 8 - int(math.Mod(float64(h.Length), 8))

	// seek to the end of the header
	if o > 0 && o < 8 {
		pad := make([]byte, o)
		n, err = r.Read(pad)
		if err != nil {
			return nil, fmt.Errorf("Error seeking beyond header padding of %d bytes: %v", o, err)
		}

		if n != o {
			return nil, fmt.Errorf("Error seeking beyond header padding of %d bytes: only %d bytes returned", o, n)
		}
	}

	return h, nil
}
<commit_msg>Readall instead of read method. Fix working with some rpms.<commit_after>package rpm

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
	"math"
)

// A Header stores metadata about a rpm package.
type Header struct {
	Version    int
	IndexCount int
	Length     int
	Indexes    IndexEntries
	Start      int
	End        int
}

// Headers is an array of Header structs.
type Headers []Header

// Predefined sizing constraints.
const (
	// MAX_HEADER_SIZE is the maximum allowable header size in bytes (32 MB).
	MAX_HEADER_SIZE = 33554432
)

// Predefined header errors.
var (
	// ErrBadHeaderLength indicates that the read header section is not the
	// expected length.
	ErrBadHeaderLength = fmt.Errorf("RPM header section is incorrect length")

	// ErrNotHeader indicates that the read header section does start with the
	// expected descriptor.
	ErrNotHeader = fmt.Errorf("invalid RPM header descriptor")

	// ErrBadStoreLength indicates that the read header store section is not the
	// expected length.
	ErrBadStoreLength = fmt.Errorf("header value store is incorrect length")
)

// Predefined header index errors.
var (
	// ErrBadIndexCount indicates that number of indexes given in the read
	// header would exceed the actual size of the header.
	ErrBadIndexCount = fmt.Errorf("index count exceeds header size")

	// ErrBadIndexLength indicates that the read header index section is not the
	// expected length.
	ErrBadIndexLength = fmt.Errorf("index section is incorrect length")

	// ErrIndexOutOfRange indicates that the read header index would exceed the
	// range of the header.
	ErrIndexOutOfRange = fmt.Errorf("index is out of range")

	// ErrBadIndexType indicates that the read index contains a value of an
	// unsupported data type.
	ErrBadIndexType = fmt.Errorf("unknown index data type")

	// ErrBadIndexValueCount indicates that the read index value would exceed
	// the range of the header store section.
	ErrBadIndexValueCount = fmt.Errorf("index value count is out of range")
)

// ReadPackageHeader reads an RPM package file header structure from the given
// io.Reader.
//
// This function should only be used if you intend to read a package header
// structure in isolation.
func ReadPackageHeader(r io.Reader) (*Header, error) {
	// read the "header structure header"
	header := make([]byte, 16)
	n, err := r.Read(header)
	if err != nil {
		return nil, err
	}

	if n != 16 {
		return nil, ErrBadHeaderLength
	}

	// check magic number
	if 0 != bytes.Compare(header[:3], []byte{0x8E, 0xAD, 0xE8}) {
		return nil, ErrNotHeader
	}

	// translate header
	h := &Header{
		Version:    int(header[3]),
		IndexCount: int(binary.BigEndian.Uint32(header[8:12])),
		Length:     int(binary.BigEndian.Uint32(header[12:16])),
	}

	// make sure header size is in range
	if h.Length > MAX_HEADER_SIZE {
		return nil, ErrBadHeaderLength
	}

	// Ensure index count is in range
	// This test is not entirely precise as h.Length also includes the value
	// store. It should at least help eliminate excessive buffer allocations for
	// corrupted length values in the > h.Length ranges.
	if h.IndexCount*16 > h.Length {
		return nil, ErrBadIndexCount
	}

	h.Indexes = make(IndexEntries, h.IndexCount)

	// read indexes
	indexLength := 16 * h.IndexCount
	indexes := make([]byte, indexLength)
	n, err = r.Read(indexes)
	if err != nil {
		return nil, err
	}

	if n != indexLength {
		return nil, ErrBadIndexLength
	}

	for x := 0; x < h.IndexCount; x++ {
		o := 16 * x
		index := IndexEntry{
			Tag:       int(binary.BigEndian.Uint32(indexes[o : o+4])),
			Type:      int(binary.BigEndian.Uint32(indexes[o+4 : o+8])),
			Offset:    int(binary.BigEndian.Uint32(indexes[o+8 : o+12])),
			ItemCount: int(binary.BigEndian.Uint32(indexes[o+12 : o+16])),
		}

		// validate index offset
		if index.Offset >= h.Length {
			return nil, ErrIndexOutOfRange
		}

		// append
		h.Indexes[x] = index
	}

	// read the "store"
	store, err := ioutil.ReadAll(r)

	if err != nil {
		return nil, err
	}

	n = len(store)

	if n != h.Length {
		return nil, ErrBadStoreLength
	}

	// parse the value of each index from the store
	for x := 0; x < h.IndexCount; x++ {
		index := h.Indexes[x]
		o := index.Offset

		if index.ItemCount == 0 {
			return nil, ErrBadIndexValueCount
		}

		switch index.Type {
		case IndexDataTypeChar:
			vals := make([]uint8, index.ItemCount)
			for v := 0; v < index.ItemCount; v++ {
				if o >= len(store) {
					return nil, fmt.Errorf("uint8 value for index %d is out of range", x+1)
				}

				vals[v] = uint8(store[o])
				o += 1
			}

			index.Value = vals

		case IndexDataTypeInt8:
			vals := make([]int8, index.ItemCount)
			for v := 0; v < index.ItemCount; v++ {
				if o >= len(store) {
					return nil, fmt.Errorf("int8 value for index %d is out of range", x+1)
				}

				vals[v] = int8(store[o])
				o += 1
			}

			index.Value = vals

		case IndexDataTypeInt16:
			vals := make([]int16, index.ItemCount)
			for v := 0; v < index.ItemCount; v++ {
				if o+2 > len(store) {
					return nil, fmt.Errorf("int16 value for index %d is out of range", x+1)
				}

				vals[v] = int16(binary.BigEndian.Uint16(store[o : o+2]))
				o += 2
			}

			index.Value = vals

		case IndexDataTypeInt32:
			vals := make([]int32, index.ItemCount)
			for v := 0; v < index.ItemCount; v++ {
				if o+4 > len(store) {
					return nil, fmt.Errorf("int32 value for index %d is out of range", x+1)
				}

				vals[v] = int32(binary.BigEndian.Uint32(store[o : o+4]))
				o += 4
			}

			index.Value = vals

		case IndexDataTypeInt64:
			vals := make([]int64, index.ItemCount)
			for v := 0; v < index.ItemCount; v++ {
				if o+8 > len(store) {
					return nil, fmt.Errorf("int64 value for index %d is out of range", x+1)
				}

				vals[v] = int64(binary.BigEndian.Uint64(store[o : o+8]))
				o += 8
			}

			index.Value = vals

		case IndexDataTypeBinary:
			if o+index.ItemCount > len(store) {
				return nil, fmt.Errorf("[]byte value for index %d is out of range", x+1)
			}

			b := make([]byte, index.ItemCount)
			copy(b, store[o:o+index.ItemCount])

			index.Value = b

		case IndexDataTypeString, IndexDataTypeStringArray, IndexDataTypeI8NString:
			// allow atleast one byte per string
			if o+index.ItemCount > len(store) {
				return nil, fmt.Errorf("[]string value for index %d is out of range", x+1)
			}

			vals := make([]string, index.ItemCount)

			for s := 0; s < index.ItemCount; s++ {
				// calculate string length
				var j int
				for j = 0; (o+j) < len(store) && store[o+j] != 0; j++ {
				}

				if j == len(store) {
					return nil, fmt.Errorf("string value for index %d is out of range", x+1)
				}

				vals[s] = string(store[o : o+j])
				o += j + 1
			}

			index.Value = vals

		case IndexDataTypeNull:
			// nothing to do here

		default:
			// unknown data type
			return nil, ErrBadIndexType
		}

		// save in array
		h.Indexes[x] = index
	}

	// calculate location of the end of the header by padding to a multiple of 8
	o := 8 - int(math.Mod(float64(h.Length), 8))

	// seek to the end of the header
	if o > 0 && o < 8 {
		pad := make([]byte, o)
		n, err = r.Read(pad)
		if err != nil {
			return nil, fmt.Errorf("Error seeking beyond header padding of %d bytes: %v", o, err)
		}

		if n != o {
			return nil, fmt.Errorf("Error seeking beyond header padding of %d bytes: only %d bytes returned", o, n)
		}
	}

	return h, nil
}
<|endoftext|>


<commit_before>package hstspreload

import (
	"fmt"
	"strconv"
	"strings"
)

const (
	// 18 weeks
	hstsMinimumMaxAge = 10886400 // seconds

	tenYears = 86400 * 365 * 10 // seconds
)

// MaxAge holds the max-age of an HSTS header in seconds.
// See https://tools.ietf.org/html/rfc6797#section-6.1.1
type MaxAge struct {
	Seconds uint64 `json:"seconds"`
}

// An HSTSHeader stores the semantics of an HSTS header.
// https://tools.ietf.org/html/rfc6797#section-6.1
//
// Note that the `preload` directive is not standardized yet:  https://crbug.com/591212
type HSTSHeader struct {
	// A MaxAge of `nil` indicates "not present".
	MaxAge            *MaxAge `json:"max_age,omitempty"`
	IncludeSubDomains bool    `json:"includeSubDomains"`
	Preload           bool    `json:"preload"`
}

// Iff Issues has no errors, the output integer is the max-age in seconds.
func parseMaxAge(directive string) (*MaxAge, Issues) {
	issues := Issues{}
	maxAgeNumericalString := directive[8:]

	// TODO: Use more concise validation code to parse a digit string to a signed int.
	for i, c := range maxAgeNumericalString {
		if i == 0 && c == '0' && len(maxAgeNumericalString) > 1 {
			issues = issues.addWarningf(
				"header.parse.max_age.leading_zero",
				"Unexpected max-age syntax",
				"The header's max-age value contains a leading 0: `%s`", directive)
		}
		if c < '0' || c > '9' {
			return nil, issues.addErrorf(
				"header.parse.max_age.non_digit_characters",
				"Invalid max-age syntax",
				"The header's max-age value contains characters that are not digits: `%s`", directive)
		}
	}

	seconds, err := strconv.ParseUint(maxAgeNumericalString, 10, 64)

	if err != nil {
		return nil, issues.addErrorf(
			"header.parse.max_age.parse_int_error",
			"Invalid max-age syntax",
			"We could not parse the header's max-age value `%s`.", maxAgeNumericalString)
	}

	return &MaxAge{Seconds: seconds}, issues
}

// ParseHeaderString parses an HSTS header. ParseHeaderString will
// report syntax errors and warnings, but does NOT calculate whether the
// header value is semantically valid. (See PreloadableHeaderString() for
// that.)
//
// To interpret the Issues that are returned, see the list of
// conventions in the documentation for Issues.
func ParseHeaderString(headerString string) (HSTSHeader, Issues) {
	hstsHeader := HSTSHeader{}
	issues := Issues{}

	directives := strings.Split(headerString, ";")
	for i, directive := range directives {
		// TODO: this trims more than spaces and tabs (LWS). https://crbug.com/596561#c10
		directives[i] = strings.TrimSpace(directive)
	}

	// If strings.Split() is given whitespace, it still returns an (empty) directive.
	// So we handle this case separately.
	if len(directives) == 1 && directives[0] == "" {
		// Return immediately, because all the extra information is redundant.
		return hstsHeader, issues.addWarningf(
			"header.parse.empty",
			"Empty Header",
			"The HSTS header is empty.")
	}

	for _, directive := range directives {
		directiveEqualsIgnoringCase := func(s string) bool {
			return strings.EqualFold(directive, s)
		}

		directiveHasPrefixIgnoringCase := func(prefix string) bool {
			return strings.HasPrefix(strings.ToLower(directive), strings.ToLower(prefix))
		}

		switch {
		case directiveEqualsIgnoringCase("preload"):
			if hstsHeader.Preload {
				issues = issues.addUniqueWarningf(
					"header.parse.repeated.preload",
					"Repeated preload directive",
					"Header contains a repeated directive: `preload`")
			} else {
				hstsHeader.Preload = true
			}

		case directiveHasPrefixIgnoringCase("preload"):
			issues = issues.addUniqueWarningf(
				"header.parse.invalid.preload",
				"Invalid preload directive",
				"Header contains a `preload` directive with extra parts.")

		case directiveEqualsIgnoringCase("includeSubDomains"):
			if hstsHeader.IncludeSubDomains {
				issues = issues.addUniqueWarningf(
					"header.parse.repeated.include_sub_domains",
					"Repeated includeSubDomains directive",
					"Header contains a repeated directive: `includeSubDomains`")
			} else {
				hstsHeader.IncludeSubDomains = true
				if directive != "includeSubDomains" {
					issues = issues.addUniqueWarningf(
						"header.parse.spelling.include_sub_domains",
						"Non-standard capitalization of includeSubDomains",
						"Header contains the token `%s`. The recommended capitalization is `includeSubDomains`.",
						directive,
					)
				}
			}

		case directiveHasPrefixIgnoringCase("includeSubDomains"):
			issues = issues.addUniqueWarningf(
				"header.parse.invalid.include_sub_domains",
				"Invalid includeSubDomains directive",
				"The header contains an `includeSubDomains` directive with extra directives.")

		case directiveHasPrefixIgnoringCase("max-age="):
			maxAge, maxAgeIssues := parseMaxAge(directive)
			issues = combineIssues(issues, maxAgeIssues)

			if len(maxAgeIssues.Errors) > 0 {
				continue
			}

			if hstsHeader.MaxAge == nil {
				hstsHeader.MaxAge = maxAge
			} else {
				issues = issues.addUniqueWarningf(
					"header.parse.repeated.max_age",
					"Repeated max-age directive",
					"The header contains a repeated directive: `max-age`")
			}

		case directiveHasPrefixIgnoringCase("max-age"):
			issues = issues.addUniqueErrorf(
				"header.parse.invalid.max_age.no_value",
				"Max-age drective without a value",
				"The header contains a max-age directive name without an associated value. Please specify the max-age in seconds.")

		case directiveEqualsIgnoringCase(""):
			issues = issues.addUniqueWarningf(
				"header.parse.empty_directive",
				"Empty directive or extra semicolon",
				"The header includes an empty directive or extra semicolon.")

		default:
			issues = issues.addWarningf(
				"header.parse.unknown_directive",
				"Unknown directive",
				"The header contains an unknown directive: `%s`", directive)
		}
	}
	return hstsHeader, issues
}

func preloadableHeaderPreload(hstsHeader HSTSHeader) Issues {
	issues := Issues{}

	if !hstsHeader.Preload {
		issues = issues.addErrorf(
			"header.preloadable.preload.missing",
			"No preload directive",
			"The header must contain the `preload` directive.")
	}

	return issues
}

func preloadableHeaderSubDomains(hstsHeader HSTSHeader) Issues {
	issues := Issues{}

	if !hstsHeader.IncludeSubDomains {
		issues = issues.addErrorf(
			"header.preloadable.include_sub_domains.missing",
			"No includeSubDomains directive",
			"The header must contain the `includeSubDomains` directive.")
	}

	return issues
}

func preloadableHeaderMaxAge(hstsHeader HSTSHeader) Issues {
	issues := Issues{}

	switch {
	case hstsHeader.MaxAge == nil:
		issues = issues.addErrorf(
			"header.preloadable.max_age.missing",
			"No max-age directice",
			"Header requirement error: Header must contain a valid `max-age` directive.")

	case hstsHeader.MaxAge.Seconds < 0:
		issues = issues.addErrorf(
			"internal.header.preloadable.max_age.negative",
			"Negative max-age",
			"Encountered an HSTSHeader with a negative max-age that does not equal MaxAgeNotPresent: %d", hstsHeader.MaxAge.Seconds)

	case hstsHeader.MaxAge.Seconds < hstsMinimumMaxAge:
		errorStr := fmt.Sprintf(
			"The max-age must be at least 10886400 seconds (== 18 weeks), but the header currently only has max-age=%d.",
			hstsHeader.MaxAge.Seconds,
		)
		if hstsHeader.MaxAge.Seconds == 0 {
			errorStr += " If you are trying to remove this domain from the preload list, please contact Lucas Garron at hstspreload@chromium.org"
			issues = issues.addErrorf(
				"header.preloadable.max_age.zero",
				"Max-age is 0",
				errorStr,
			)
		} else {
			issues = issues.addErrorf(
				"header.preloadable.max_age.too_low",
				"Max-age too low",
				errorStr,
			)
		}

	case hstsHeader.MaxAge.Seconds > tenYears:
		issues = issues.addWarningf(
			"header.preloadable.max_age.over_10_years",
			"Max-age > 10 years",
			"FYI: The max-age (%d seconds) is longer than 10 years, which is an unusually long value.",
			hstsHeader.MaxAge.Seconds,
		)

	}

	return issues
}

// PreloadableHeader checks whether hstsHeader satisfies all requirements
// for preloading in Chromium.
//
// To interpret the result, see the list of conventions in the
// documentation for Issues.
//
// Most of the time, you'll probably want to use PreloadableHeaderString() instead.
func PreloadableHeader(hstsHeader HSTSHeader) Issues {
	issues := Issues{}

	issues = combineIssues(issues, preloadableHeaderSubDomains(hstsHeader))
	issues = combineIssues(issues, preloadableHeaderPreload(hstsHeader))
	issues = combineIssues(issues, preloadableHeaderMaxAge(hstsHeader))
	return issues
}

// RemovableHeader checks whether the header satisfies all requirements
// for being removed from the Chromium preload list.
//
// To interpret the result, see the list of conventions in the
// documentation for Issues.
//
// Most of the time, you'll probably want to use RemovableHeaderString() instead.
func RemovableHeader(hstsHeader HSTSHeader) Issues {
	issues := Issues{}

	if hstsHeader.Preload {
		issues = issues.addErrorf(
			"header.removable.contains.preload",
			"Contains preload directive",
			"Header requirement error: For preload list removal, the header must not contain the `preload` directive.")
	}

	if hstsHeader.MaxAge == nil {
		issues = issues.addErrorf(
			"header.removable.missing.max_age",
			"No max-age directive",
			"Header requirement error: Header must contain a valid `max-age` directive.")
	}

	return issues
}

// PreloadableHeaderString is a convenience function that calls
// ParseHeaderString() and then calls on PreloadableHeader() the parsed
// header. It returns all issues from both calls, combined.
//
// To interpret the result, see the list of conventions in the
// documentation for Issues.
func PreloadableHeaderString(headerString string) Issues {
	hstsHeader, issues := ParseHeaderString(headerString)
	return combineIssues(issues, PreloadableHeader(hstsHeader))
}

// RemovableHeaderString is a convenience function that calls
// ParseHeaderString() and then calls on RemovableHeader() the parsed
// header. It returns all errors from ParseHeaderString() and all
// issues from RemovableHeader(). Note that *warnings* from
// ParseHeaderString() are ignored, since domains asking to be removed
// will often have minor errors that shouldn't affect removal. It's
// better to have a cleaner verdict in this case.
//
// To interpret the result, see the list of conventions in the
// documentation for Issues.
func RemovableHeaderString(headerString string) Issues {
	hstsHeader, issues := ParseHeaderString(headerString)
	issues = Issues{
		Errors: issues.Errors,
		// Ignore parse warnings for removal testing.
	}
	return combineIssues(issues, RemovableHeader(hstsHeader))
}
<commit_msg>Make a note about quoted values in parseMaxAge().<commit_after>package hstspreload

import (
	"fmt"
	"strconv"
	"strings"
)

const (
	// 18 weeks
	hstsMinimumMaxAge = 10886400 // seconds

	tenYears = 86400 * 365 * 10 // seconds
)

// MaxAge holds the max-age of an HSTS header in seconds.
// See https://tools.ietf.org/html/rfc6797#section-6.1.1
type MaxAge struct {
	Seconds uint64 `json:"seconds"`
}

// An HSTSHeader stores the semantics of an HSTS header.
// https://tools.ietf.org/html/rfc6797#section-6.1
//
// Note that the `preload` directive is not standardized yet:  https://crbug.com/591212
type HSTSHeader struct {
	// A MaxAge of `nil` indicates "not present".
	MaxAge            *MaxAge `json:"max_age,omitempty"`
	IncludeSubDomains bool    `json:"includeSubDomains"`
	Preload           bool    `json:"preload"`
}

// Iff Issues has no errors, the output integer is the max-age in seconds.
// Note that according to the spec, the max-age value may optionally be quoted:
// https://tools.ietf.org/html/rfc6797#section-6.2
// However, it seems no one does this in practice, and certainly no one has
// asked to be preloaded with a quoted max-age value. So to keep things simple,
// we don't support quoted values.
func parseMaxAge(directive string) (*MaxAge, Issues) {
	issues := Issues{}
	maxAgeNumericalString := directive[8:]

	// TODO: Use more concise validation code to parse a digit string to a signed int.
	for i, c := range maxAgeNumericalString {
		if i == 0 && c == '0' && len(maxAgeNumericalString) > 1 {
			issues = issues.addWarningf(
				"header.parse.max_age.leading_zero",
				"Unexpected max-age syntax",
				"The header's max-age value contains a leading 0: `%s`", directive)
		}
		if c < '0' || c > '9' {
			return nil, issues.addErrorf(
				"header.parse.max_age.non_digit_characters",
				"Invalid max-age syntax",
				"The header's max-age value contains characters that are not digits: `%s`", directive)
		}
	}

	seconds, err := strconv.ParseUint(maxAgeNumericalString, 10, 64)

	if err != nil {
		return nil, issues.addErrorf(
			"header.parse.max_age.parse_int_error",
			"Invalid max-age syntax",
			"We could not parse the header's max-age value `%s`.", maxAgeNumericalString)
	}

	return &MaxAge{Seconds: seconds}, issues
}

// ParseHeaderString parses an HSTS header. ParseHeaderString will
// report syntax errors and warnings, but does NOT calculate whether the
// header value is semantically valid. (See PreloadableHeaderString() for
// that.)
//
// To interpret the Issues that are returned, see the list of
// conventions in the documentation for Issues.
func ParseHeaderString(headerString string) (HSTSHeader, Issues) {
	hstsHeader := HSTSHeader{}
	issues := Issues{}

	directives := strings.Split(headerString, ";")
	for i, directive := range directives {
		// TODO: this trims more than spaces and tabs (LWS). https://crbug.com/596561#c10
		directives[i] = strings.TrimSpace(directive)
	}

	// If strings.Split() is given whitespace, it still returns an (empty) directive.
	// So we handle this case separately.
	if len(directives) == 1 && directives[0] == "" {
		// Return immediately, because all the extra information is redundant.
		return hstsHeader, issues.addWarningf(
			"header.parse.empty",
			"Empty Header",
			"The HSTS header is empty.")
	}

	for _, directive := range directives {
		directiveEqualsIgnoringCase := func(s string) bool {
			return strings.EqualFold(directive, s)
		}

		directiveHasPrefixIgnoringCase := func(prefix string) bool {
			return strings.HasPrefix(strings.ToLower(directive), strings.ToLower(prefix))
		}

		switch {
		case directiveEqualsIgnoringCase("preload"):
			if hstsHeader.Preload {
				issues = issues.addUniqueWarningf(
					"header.parse.repeated.preload",
					"Repeated preload directive",
					"Header contains a repeated directive: `preload`")
			} else {
				hstsHeader.Preload = true
			}

		case directiveHasPrefixIgnoringCase("preload"):
			issues = issues.addUniqueWarningf(
				"header.parse.invalid.preload",
				"Invalid preload directive",
				"Header contains a `preload` directive with extra parts.")

		case directiveEqualsIgnoringCase("includeSubDomains"):
			if hstsHeader.IncludeSubDomains {
				issues = issues.addUniqueWarningf(
					"header.parse.repeated.include_sub_domains",
					"Repeated includeSubDomains directive",
					"Header contains a repeated directive: `includeSubDomains`")
			} else {
				hstsHeader.IncludeSubDomains = true
				if directive != "includeSubDomains" {
					issues = issues.addUniqueWarningf(
						"header.parse.spelling.include_sub_domains",
						"Non-standard capitalization of includeSubDomains",
						"Header contains the token `%s`. The recommended capitalization is `includeSubDomains`.",
						directive,
					)
				}
			}

		case directiveHasPrefixIgnoringCase("includeSubDomains"):
			issues = issues.addUniqueWarningf(
				"header.parse.invalid.include_sub_domains",
				"Invalid includeSubDomains directive",
				"The header contains an `includeSubDomains` directive with extra directives.")

		case directiveHasPrefixIgnoringCase("max-age="):
			maxAge, maxAgeIssues := parseMaxAge(directive)
			issues = combineIssues(issues, maxAgeIssues)

			if len(maxAgeIssues.Errors) > 0 {
				continue
			}

			if hstsHeader.MaxAge == nil {
				hstsHeader.MaxAge = maxAge
			} else {
				issues = issues.addUniqueWarningf(
					"header.parse.repeated.max_age",
					"Repeated max-age directive",
					"The header contains a repeated directive: `max-age`")
			}

		case directiveHasPrefixIgnoringCase("max-age"):
			issues = issues.addUniqueErrorf(
				"header.parse.invalid.max_age.no_value",
				"Max-age drective without a value",
				"The header contains a max-age directive name without an associated value. Please specify the max-age in seconds.")

		case directiveEqualsIgnoringCase(""):
			issues = issues.addUniqueWarningf(
				"header.parse.empty_directive",
				"Empty directive or extra semicolon",
				"The header includes an empty directive or extra semicolon.")

		default:
			issues = issues.addWarningf(
				"header.parse.unknown_directive",
				"Unknown directive",
				"The header contains an unknown directive: `%s`", directive)
		}
	}
	return hstsHeader, issues
}

func preloadableHeaderPreload(hstsHeader HSTSHeader) Issues {
	issues := Issues{}

	if !hstsHeader.Preload {
		issues = issues.addErrorf(
			"header.preloadable.preload.missing",
			"No preload directive",
			"The header must contain the `preload` directive.")
	}

	return issues
}

func preloadableHeaderSubDomains(hstsHeader HSTSHeader) Issues {
	issues := Issues{}

	if !hstsHeader.IncludeSubDomains {
		issues = issues.addErrorf(
			"header.preloadable.include_sub_domains.missing",
			"No includeSubDomains directive",
			"The header must contain the `includeSubDomains` directive.")
	}

	return issues
}

func preloadableHeaderMaxAge(hstsHeader HSTSHeader) Issues {
	issues := Issues{}

	switch {
	case hstsHeader.MaxAge == nil:
		issues = issues.addErrorf(
			"header.preloadable.max_age.missing",
			"No max-age directice",
			"Header requirement error: Header must contain a valid `max-age` directive.")

	case hstsHeader.MaxAge.Seconds < 0:
		issues = issues.addErrorf(
			"internal.header.preloadable.max_age.negative",
			"Negative max-age",
			"Encountered an HSTSHeader with a negative max-age that does not equal MaxAgeNotPresent: %d", hstsHeader.MaxAge.Seconds)

	case hstsHeader.MaxAge.Seconds < hstsMinimumMaxAge:
		errorStr := fmt.Sprintf(
			"The max-age must be at least 10886400 seconds (== 18 weeks), but the header currently only has max-age=%d.",
			hstsHeader.MaxAge.Seconds,
		)
		if hstsHeader.MaxAge.Seconds == 0 {
			errorStr += " If you are trying to remove this domain from the preload list, please contact Lucas Garron at hstspreload@chromium.org"
			issues = issues.addErrorf(
				"header.preloadable.max_age.zero",
				"Max-age is 0",
				errorStr,
			)
		} else {
			issues = issues.addErrorf(
				"header.preloadable.max_age.too_low",
				"Max-age too low",
				errorStr,
			)
		}

	case hstsHeader.MaxAge.Seconds > tenYears:
		issues = issues.addWarningf(
			"header.preloadable.max_age.over_10_years",
			"Max-age > 10 years",
			"FYI: The max-age (%d seconds) is longer than 10 years, which is an unusually long value.",
			hstsHeader.MaxAge.Seconds,
		)

	}

	return issues
}

// PreloadableHeader checks whether hstsHeader satisfies all requirements
// for preloading in Chromium.
//
// To interpret the result, see the list of conventions in the
// documentation for Issues.
//
// Most of the time, you'll probably want to use PreloadableHeaderString() instead.
func PreloadableHeader(hstsHeader HSTSHeader) Issues {
	issues := Issues{}

	issues = combineIssues(issues, preloadableHeaderSubDomains(hstsHeader))
	issues = combineIssues(issues, preloadableHeaderPreload(hstsHeader))
	issues = combineIssues(issues, preloadableHeaderMaxAge(hstsHeader))
	return issues
}

// RemovableHeader checks whether the header satisfies all requirements
// for being removed from the Chromium preload list.
//
// To interpret the result, see the list of conventions in the
// documentation for Issues.
//
// Most of the time, you'll probably want to use RemovableHeaderString() instead.
func RemovableHeader(hstsHeader HSTSHeader) Issues {
	issues := Issues{}

	if hstsHeader.Preload {
		issues = issues.addErrorf(
			"header.removable.contains.preload",
			"Contains preload directive",
			"Header requirement error: For preload list removal, the header must not contain the `preload` directive.")
	}

	if hstsHeader.MaxAge == nil {
		issues = issues.addErrorf(
			"header.removable.missing.max_age",
			"No max-age directive",
			"Header requirement error: Header must contain a valid `max-age` directive.")
	}

	return issues
}

// PreloadableHeaderString is a convenience function that calls
// ParseHeaderString() and then calls on PreloadableHeader() the parsed
// header. It returns all issues from both calls, combined.
//
// To interpret the result, see the list of conventions in the
// documentation for Issues.
func PreloadableHeaderString(headerString string) Issues {
	hstsHeader, issues := ParseHeaderString(headerString)
	return combineIssues(issues, PreloadableHeader(hstsHeader))
}

// RemovableHeaderString is a convenience function that calls
// ParseHeaderString() and then calls on RemovableHeader() the parsed
// header. It returns all errors from ParseHeaderString() and all
// issues from RemovableHeader(). Note that *warnings* from
// ParseHeaderString() are ignored, since domains asking to be removed
// will often have minor errors that shouldn't affect removal. It's
// better to have a cleaner verdict in this case.
//
// To interpret the result, see the list of conventions in the
// documentation for Issues.
func RemovableHeaderString(headerString string) Issues {
	hstsHeader, issues := ParseHeaderString(headerString)
	issues = Issues{
		Errors: issues.Errors,
		// Ignore parse warnings for removal testing.
	}
	return combineIssues(issues, RemovableHeader(hstsHeader))
}
<|endoftext|>