<commit_before>package bundler

// This test file contains tests on checking the correctness of BundleFromFile and Bundle.
// We simulate various scenarios for Bundle and funnel the tests through BundleFromFile.
import (
	"encoding/json"
	"testing"
)

// A helper structure that defines a BundleFromFile test case.
type fileTest struct {
	// PEM cert file to be bundled
	cert string
	// PEM private key file to be bundled
	key string
	// Root CA bundle
	caBundleFile string
	// Trust intermediate bundle
	intBundleFile string
	// Additional PEM intermediate certificates to be added into the bundler
	extraIntermediates string
	// Bundler creation function
	bundlerConstructor func(*testing.T) (b *Bundler)
	// Error checking function
	errorCallback func(*testing.T, error)
	// Bundle checking function
	bundleChecking func(*testing.T, *Bundle)
}

/* ========== BundleFromFile Test Setup =============

For each pair of crypto algorithm X and key size Y, a CA chain is constructed:
	Test_root_CA -> inter-L1 -> inter-L2--> cfssl-leaf-ecdsa256
	                                    |-> cfssl-leaf-ecdsa384
	                                    |-> cfssl-leaf-ecdsa521
	                                    |-> cfssl-leaf-rsa2048
	                                    |-> cfssl-leaf-rsa3072
	                                    |-> cfssl-leaf-rsa4096

Test_root_CA is a RSA cert, inter-L1 is RSA 4096 cert, inter-L2 is ecdsa-384 cert.

The max path length is set to be 1 for non-root CAs.
Two inter-* certs are assembled in intermediates.crt

There is also an expired L1 cert, sharing the same CSR with inter-L1. Also the
root CA processes the inter-L2 CSR directly to generate inter-L2-direct cert.
*	Test_root_CA--> inter-L1-expired
	            |-> inter-L2-direct
Using inter-L2-direct as additional intermediate cert should shorten the
bundle chain.
*/
const (
	leafECDSA256    = "testdata/cfssl-leaf-ecdsa256.pem"
	leafECDSA384    = "testdata/cfssl-leaf-ecdsa384.pem"
	leafECDSA521    = "testdata/cfssl-leaf-ecdsa521.pem"
	leafRSA2048     = "testdata/cfssl-leaf-rsa2048.pem"
	leafRSA3072     = "testdata/cfssl-leaf-rsa3072.pem"
	leafRSA4096     = "testdata/cfssl-leaf-rsa4096.pem"
	leafKeyECDSA256 = "testdata/cfssl-leaf-ecdsa256.key"
	leafKeyECDSA384 = "testdata/cfssl-leaf-ecdsa384.key"
	leafKeyECDSA521 = "testdata/cfssl-leaf-ecdsa521.key"
	leafKeyRSA2048  = "testdata/cfssl-leaf-rsa2048.key"
	leafKeyRSA3072  = "testdata/cfssl-leaf-rsa3072.key"
	leafKeyRSA4096  = "testdata/cfssl-leaf-rsa4096.key"
	leafletRSA4096  = "testdata/cfssl-leaflet-rsa4096.pem"
	interL1         = "testdata/inter-L1.pem"
	interL1Expired  = "testdata/inter-L1-expired.pem"
	interL1CSR      = "testdata/inter-L1.csr"
	interL2         = "testdata/inter-L2.pem"

	interL2Direct = "testdata/inter-L2-direct.pem"
	partialBundle = "testdata/partial-bundle.pem"         // partialBundle is a partial cert chain {leaf-ecds256,  inter-L2}
	rpBundle      = "testdata/reverse-partial-bundle.pem" // partialBundle is a partial cert chain in the reverse order {inter-L2, leaf-ecdsa256}
	badBundle     = "testdata/bad-bundle.pem"             // badBundle is a non-verifying partial bundle {leaf-ecdsa256, leaf-ecdsa384}
	interL2CSR    = "testdata/inter-L2.csr"
	certDSA2048   = "testdata/dsa2048.pem"
	keyDSA2048    = "testdata/dsa2048.key"
)

// BundleFromFile test cases.
var fileTests = []fileTest{
	// Input verification
	{
		cert:          "not_such_cert.pem",
		caBundleFile:  testCFSSLRootBundle,
		intBundleFile: testCFSSLIntBundle,
		errorCallback: ExpectErrorMessage(`"code":1001`),
	},
	{
		cert:          emptyPEM,
		caBundleFile:  testCFSSLRootBundle,
		intBundleFile: testCFSSLIntBundle,
		errorCallback: ExpectErrorMessage(`"code":1002`),
	},

	// Normal Keyless bundling for all supported public key types
	{
		cert:           leafECDSA256,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  testCFSSLIntBundle,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},
	{
		cert:           leafECDSA384,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  testCFSSLIntBundle,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},
	{
		cert:           leafECDSA521,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  testCFSSLIntBundle,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},
	{
		cert:           leafRSA2048,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  testCFSSLIntBundle,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},
	{
		cert:           leafRSA3072,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  testCFSSLIntBundle,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},
	{
		cert:           leafRSA4096,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  testCFSSLIntBundle,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},

	// Normal bundling with private key for all supported key types
	{
		cert:           leafECDSA256,
		key:            leafKeyECDSA256,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  testCFSSLIntBundle,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},
	{
		cert:           leafECDSA384,
		key:            leafKeyECDSA384,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  testCFSSLIntBundle,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},
	{
		cert:           leafECDSA521,
		key:            leafKeyECDSA521,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  testCFSSLIntBundle,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},
	{
		cert:           leafRSA2048,
		key:            leafKeyRSA2048,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  testCFSSLIntBundle,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},
	{
		cert:           leafRSA3072,
		key:            leafKeyRSA3072,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  testCFSSLIntBundle,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},
	{
		cert:           leafRSA4096,
		key:            leafKeyRSA4096,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  testCFSSLIntBundle,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},

	// Bundling with errors

	// leaflet cert is signed by a leaf cert which is not included the intermediate bundle.
	// So an UnknownAuthority error is expected.
	{
		cert:          leafletRSA4096,
		caBundleFile:  testCFSSLRootBundle,
		intBundleFile: testCFSSLIntBundle,
		errorCallback: ExpectErrorMessage(`"code":1220`),
	},
	// Expect TooManyIntermediates error because max path length is 1 for
	// inter-L1 but the leaflet cert is 2 CA away from inter-L1.
	{
		cert:               leafletRSA4096,
		extraIntermediates: leafRSA4096,
		caBundleFile:       testCFSSLRootBundle,
		intBundleFile:      testCFSSLIntBundle,
		errorCallback:      ExpectErrorMessage(`"code":1213`),
	},
	// Bundle with expired inter-L1 intermediate cert only, expect error 1211 VerifyFailed:Expired.
	{
		cert:               interL2,
		extraIntermediates: interL1Expired,
		caBundleFile:       testCFSSLRootBundle,
		intBundleFile:      emptyPEM,
		errorCallback:      ExpectErrorMessage(`"code":1211`),
	},

	// Bundle with private key mismatch
	// RSA cert, ECC private key
	{
		cert:          leafRSA4096,
		key:           leafKeyECDSA256,
		caBundleFile:  testCFSSLRootBundle,
		intBundleFile: testCFSSLIntBundle,
		errorCallback: ExpectErrorMessages([]string{`"code":2300,`, `"message":"Private key does not match public key"`}),
	},
	// ECC cert, RSA private key
	{
		cert:          leafECDSA256,
		key:           leafKeyRSA4096,
		caBundleFile:  testCFSSLRootBundle,
		intBundleFile: testCFSSLIntBundle,
		errorCallback: ExpectErrorMessages([]string{`"code":2300,`, `"message":"Private key does not match public key"`}),
	},
	// RSA 2048 cert, RSA 4096  private key
	{
		cert:          leafRSA2048,
		key:           leafKeyRSA4096,
		caBundleFile:  testCFSSLRootBundle,
		intBundleFile: testCFSSLIntBundle,
		errorCallback: ExpectErrorMessages([]string{`"code":2300,`, `"message":"Private key does not match public key"`}),
	},
	// ECDSA 256 cert, ECDSA 384  private key
	{
		cert:          leafECDSA256,
		key:           leafKeyECDSA384,
		caBundleFile:  testCFSSLRootBundle,
		intBundleFile: testCFSSLIntBundle,
		errorCallback: ExpectErrorMessages([]string{`"code":2300,`, `"message":"Private key does not match public key"`}),
	},

	// DSA is NOT supported.
	// Keyless bundling, expect private key error "NotRSAOrECC"
	{
		cert:          certDSA2048,
		caBundleFile:  testCFSSLRootBundle,
		intBundleFile: testCFSSLIntBundle,
		errorCallback: ExpectErrorMessages([]string{`"code":2200,`, `"message":"Private key algorithm is not RSA or ECC"`}),
	},
	// Bundling with DSA private key, expect error "Failed to parse private key"
	{
		cert:          certDSA2048,
		key:           keyDSA2048,
		caBundleFile:  testCFSSLRootBundle,
		intBundleFile: testCFSSLIntBundle,
		errorCallback: ExpectErrorMessages([]string{`"code":2003,`, `"message":"Failed to parse private key"`}),
	},

	// Bundle with partial chain less some intermediates, expected error 1220: UnknownAuthority
	{
		cert:          badBundle,
		caBundleFile:  testCFSSLRootBundle,
		intBundleFile: interL1,
		errorCallback: ExpectErrorMessage(`"code":1220`),
	},

	// Bundle with misplaced key as cert
	{
		cert:          leafKeyECDSA256,
		caBundleFile:  testCFSSLRootBundle,
		intBundleFile: testCFSSLIntBundle,
		errorCallback: ExpectErrorMessages([]string{`"code":1003,`, `"message":"Failed to parse certificate"`}),
	},

	// Bundle with misplaced cert as key
	{
		cert:          leafECDSA256,
		key:           leafECDSA256,
		caBundleFile:  testCFSSLRootBundle,
		intBundleFile: testCFSSLIntBundle,
		errorCallback: ExpectErrorMessages([]string{`"code":2003,`, `"message":"Failed to parse private key"`}),
	},

	// Smart Bundling
	// Bundling with a partial bundle should work the same as bundling the leaf.
	{
		cert:           partialBundle,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  testCFSSLIntBundle,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},

	// Bundle with a partial bundle such that the intermediate provided in the
	// partial bundle is verify by an intermediate. Yet itself is not in the intermediate
	// pool. In such cases, the bundling should be able to store the new intermediate
	// and return a correct bundle.
	{
		cert:           partialBundle,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  interL1,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},

	// Bundle with a reverse-ordered partial bundle.
	// Bundler should be able to detect it and return a correct bundle.
	{
		cert:           rpBundle,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  interL1,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},

	// Bundle with a L2 cert direct signed by root, expect a shorter chain of length 2.
	{
		cert:               leafECDSA256,
		extraIntermediates: interL2Direct,
		caBundleFile:       testCFSSLRootBundle,
		intBundleFile:      testCFSSLIntBundle,
		errorCallback:      nil,
		bundleChecking:     ExpectBundleLength(2),
	},
}

// TestBundleFromFile goes through test cases defined in fileTests. See below for test cases definition and details.
func TestBundleFromFile(t *testing.T) {
	for _, test := range fileTests {
		b := newCustomizedBundlerFromFile(t, test.caBundleFile, test.intBundleFile, test.extraIntermediates)
		bundle, err := b.BundleFromFile(test.cert, test.key, Optimal, "")
		if test.errorCallback != nil {
			test.errorCallback(t, err)
		} else {
			if err != nil {
				t.Fatalf("expected no error. but an error occurred: %v", err)
			}
			if test.bundleChecking != nil {
				test.bundleChecking(t, bundle)
			}
		}

		if bundle != nil {
			bundle.Cert = nil
			if _, err = json.Marshal(bundle); err == nil {
				t.Fatal("bundle should fail with no cert")
			}
		}
	}
}
<commit_msg>bundler: temporarily remove `leafRSA3072` testcases.<commit_after>package bundler

// This test file contains tests on checking the correctness of BundleFromFile and Bundle.
// We simulate various scenarios for Bundle and funnel the tests through BundleFromFile.
import (
	"encoding/json"
	"testing"
)

// A helper structure that defines a BundleFromFile test case.
type fileTest struct {
	// PEM cert file to be bundled
	cert string
	// PEM private key file to be bundled
	key string
	// Root CA bundle
	caBundleFile string
	// Trust intermediate bundle
	intBundleFile string
	// Additional PEM intermediate certificates to be added into the bundler
	extraIntermediates string
	// Bundler creation function
	bundlerConstructor func(*testing.T) (b *Bundler)
	// Error checking function
	errorCallback func(*testing.T, error)
	// Bundle checking function
	bundleChecking func(*testing.T, *Bundle)
}

/* ========== BundleFromFile Test Setup =============

For each pair of crypto algorithm X and key size Y, a CA chain is constructed:
	Test_root_CA -> inter-L1 -> inter-L2--> cfssl-leaf-ecdsa256
	                                    |-> cfssl-leaf-ecdsa384
	                                    |-> cfssl-leaf-ecdsa521
	                                    |-> cfssl-leaf-rsa2048
	                                    |-> cfssl-leaf-rsa3072
	                                    |-> cfssl-leaf-rsa4096

Test_root_CA is a RSA cert, inter-L1 is RSA 4096 cert, inter-L2 is ecdsa-384 cert.

The max path length is set to be 1 for non-root CAs.
Two inter-* certs are assembled in intermediates.crt

There is also an expired L1 cert, sharing the same CSR with inter-L1. Also the
root CA processes the inter-L2 CSR directly to generate inter-L2-direct cert.
*	Test_root_CA--> inter-L1-expired
	            |-> inter-L2-direct
Using inter-L2-direct as additional intermediate cert should shorten the
bundle chain.
*/
const (
	leafECDSA256    = "testdata/cfssl-leaf-ecdsa256.pem"
	leafECDSA384    = "testdata/cfssl-leaf-ecdsa384.pem"
	leafECDSA521    = "testdata/cfssl-leaf-ecdsa521.pem"
	leafRSA2048     = "testdata/cfssl-leaf-rsa2048.pem"
	leafRSA3072     = "testdata/cfssl-leaf-rsa3072.pem"
	leafRSA4096     = "testdata/cfssl-leaf-rsa4096.pem"
	leafKeyECDSA256 = "testdata/cfssl-leaf-ecdsa256.key"
	leafKeyECDSA384 = "testdata/cfssl-leaf-ecdsa384.key"
	leafKeyECDSA521 = "testdata/cfssl-leaf-ecdsa521.key"
	leafKeyRSA2048  = "testdata/cfssl-leaf-rsa2048.key"
	leafKeyRSA3072  = "testdata/cfssl-leaf-rsa3072.key"
	leafKeyRSA4096  = "testdata/cfssl-leaf-rsa4096.key"
	leafletRSA4096  = "testdata/cfssl-leaflet-rsa4096.pem"
	interL1         = "testdata/inter-L1.pem"
	interL1Expired  = "testdata/inter-L1-expired.pem"
	interL1CSR      = "testdata/inter-L1.csr"
	interL2         = "testdata/inter-L2.pem"

	interL2Direct = "testdata/inter-L2-direct.pem"
	partialBundle = "testdata/partial-bundle.pem"         // partialBundle is a partial cert chain {leaf-ecds256,  inter-L2}
	rpBundle      = "testdata/reverse-partial-bundle.pem" // partialBundle is a partial cert chain in the reverse order {inter-L2, leaf-ecdsa256}
	badBundle     = "testdata/bad-bundle.pem"             // badBundle is a non-verifying partial bundle {leaf-ecdsa256, leaf-ecdsa384}
	interL2CSR    = "testdata/inter-L2.csr"
	certDSA2048   = "testdata/dsa2048.pem"
	keyDSA2048    = "testdata/dsa2048.key"
)

// BundleFromFile test cases.
var fileTests = []fileTest{
	// Input verification
	{
		cert:          "not_such_cert.pem",
		caBundleFile:  testCFSSLRootBundle,
		intBundleFile: testCFSSLIntBundle,
		errorCallback: ExpectErrorMessage(`"code":1001`),
	},
	{
		cert:          emptyPEM,
		caBundleFile:  testCFSSLRootBundle,
		intBundleFile: testCFSSLIntBundle,
		errorCallback: ExpectErrorMessage(`"code":1002`),
	},

	// Normal Keyless bundling for all supported public key types
	{
		cert:           leafECDSA256,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  testCFSSLIntBundle,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},
	{
		cert:           leafECDSA384,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  testCFSSLIntBundle,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},
	{
		cert:           leafECDSA521,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  testCFSSLIntBundle,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},
	{
		cert:           leafRSA2048,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  testCFSSLIntBundle,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},
	/*
		TODO: Re-enable once leafRSA3072 is regenerated with new expiry.
		{
			cert:           leafRSA3072,
			caBundleFile:   testCFSSLRootBundle,
			intBundleFile:  testCFSSLIntBundle,
			errorCallback:  nil,
			bundleChecking: ExpectBundleLength(3),
		},
	*/
	{
		cert:           leafRSA4096,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  testCFSSLIntBundle,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},

	// Normal bundling with private key for all supported key types
	{
		cert:           leafECDSA256,
		key:            leafKeyECDSA256,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  testCFSSLIntBundle,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},
	{
		cert:           leafECDSA384,
		key:            leafKeyECDSA384,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  testCFSSLIntBundle,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},
	{
		cert:           leafECDSA521,
		key:            leafKeyECDSA521,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  testCFSSLIntBundle,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},
	{
		cert:           leafRSA2048,
		key:            leafKeyRSA2048,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  testCFSSLIntBundle,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},
	/*
			TODO: Re-enable once leafRSA3072 is regenerated with new expiry.
		{
			cert:           leafRSA3072,
			key:            leafKeyRSA3072,
			caBundleFile:   testCFSSLRootBundle,
			intBundleFile:  testCFSSLIntBundle,
			errorCallback:  nil,
			bundleChecking: ExpectBundleLength(3),
		},
	*/
	{
		cert:           leafRSA4096,
		key:            leafKeyRSA4096,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  testCFSSLIntBundle,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},

	// Bundling with errors

	// leaflet cert is signed by a leaf cert which is not included the intermediate bundle.
	// So an UnknownAuthority error is expected.
	{
		cert:          leafletRSA4096,
		caBundleFile:  testCFSSLRootBundle,
		intBundleFile: testCFSSLIntBundle,
		errorCallback: ExpectErrorMessage(`"code":1220`),
	},
	// Expect TooManyIntermediates error because max path length is 1 for
	// inter-L1 but the leaflet cert is 2 CA away from inter-L1.
	{
		cert:               leafletRSA4096,
		extraIntermediates: leafRSA4096,
		caBundleFile:       testCFSSLRootBundle,
		intBundleFile:      testCFSSLIntBundle,
		errorCallback:      ExpectErrorMessage(`"code":1213`),
	},
	// Bundle with expired inter-L1 intermediate cert only, expect error 1211 VerifyFailed:Expired.
	{
		cert:               interL2,
		extraIntermediates: interL1Expired,
		caBundleFile:       testCFSSLRootBundle,
		intBundleFile:      emptyPEM,
		errorCallback:      ExpectErrorMessage(`"code":1211`),
	},

	// Bundle with private key mismatch
	// RSA cert, ECC private key
	{
		cert:          leafRSA4096,
		key:           leafKeyECDSA256,
		caBundleFile:  testCFSSLRootBundle,
		intBundleFile: testCFSSLIntBundle,
		errorCallback: ExpectErrorMessages([]string{`"code":2300,`, `"message":"Private key does not match public key"`}),
	},
	// ECC cert, RSA private key
	{
		cert:          leafECDSA256,
		key:           leafKeyRSA4096,
		caBundleFile:  testCFSSLRootBundle,
		intBundleFile: testCFSSLIntBundle,
		errorCallback: ExpectErrorMessages([]string{`"code":2300,`, `"message":"Private key does not match public key"`}),
	},
	// RSA 2048 cert, RSA 4096  private key
	{
		cert:          leafRSA2048,
		key:           leafKeyRSA4096,
		caBundleFile:  testCFSSLRootBundle,
		intBundleFile: testCFSSLIntBundle,
		errorCallback: ExpectErrorMessages([]string{`"code":2300,`, `"message":"Private key does not match public key"`}),
	},
	// ECDSA 256 cert, ECDSA 384  private key
	{
		cert:          leafECDSA256,
		key:           leafKeyECDSA384,
		caBundleFile:  testCFSSLRootBundle,
		intBundleFile: testCFSSLIntBundle,
		errorCallback: ExpectErrorMessages([]string{`"code":2300,`, `"message":"Private key does not match public key"`}),
	},

	// DSA is NOT supported.
	// Keyless bundling, expect private key error "NotRSAOrECC"
	{
		cert:          certDSA2048,
		caBundleFile:  testCFSSLRootBundle,
		intBundleFile: testCFSSLIntBundle,
		errorCallback: ExpectErrorMessages([]string{`"code":2200,`, `"message":"Private key algorithm is not RSA or ECC"`}),
	},
	// Bundling with DSA private key, expect error "Failed to parse private key"
	{
		cert:          certDSA2048,
		key:           keyDSA2048,
		caBundleFile:  testCFSSLRootBundle,
		intBundleFile: testCFSSLIntBundle,
		errorCallback: ExpectErrorMessages([]string{`"code":2003,`, `"message":"Failed to parse private key"`}),
	},

	// Bundle with partial chain less some intermediates, expected error 1220: UnknownAuthority
	{
		cert:          badBundle,
		caBundleFile:  testCFSSLRootBundle,
		intBundleFile: interL1,
		errorCallback: ExpectErrorMessage(`"code":1220`),
	},

	// Bundle with misplaced key as cert
	{
		cert:          leafKeyECDSA256,
		caBundleFile:  testCFSSLRootBundle,
		intBundleFile: testCFSSLIntBundle,
		errorCallback: ExpectErrorMessages([]string{`"code":1003,`, `"message":"Failed to parse certificate"`}),
	},

	// Bundle with misplaced cert as key
	{
		cert:          leafECDSA256,
		key:           leafECDSA256,
		caBundleFile:  testCFSSLRootBundle,
		intBundleFile: testCFSSLIntBundle,
		errorCallback: ExpectErrorMessages([]string{`"code":2003,`, `"message":"Failed to parse private key"`}),
	},

	// Smart Bundling
	// Bundling with a partial bundle should work the same as bundling the leaf.
	{
		cert:           partialBundle,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  testCFSSLIntBundle,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},

	// Bundle with a partial bundle such that the intermediate provided in the
	// partial bundle is verify by an intermediate. Yet itself is not in the intermediate
	// pool. In such cases, the bundling should be able to store the new intermediate
	// and return a correct bundle.
	{
		cert:           partialBundle,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  interL1,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},

	// Bundle with a reverse-ordered partial bundle.
	// Bundler should be able to detect it and return a correct bundle.
	{
		cert:           rpBundle,
		caBundleFile:   testCFSSLRootBundle,
		intBundleFile:  interL1,
		errorCallback:  nil,
		bundleChecking: ExpectBundleLength(3),
	},

	// Bundle with a L2 cert direct signed by root, expect a shorter chain of length 2.
	{
		cert:               leafECDSA256,
		extraIntermediates: interL2Direct,
		caBundleFile:       testCFSSLRootBundle,
		intBundleFile:      testCFSSLIntBundle,
		errorCallback:      nil,
		bundleChecking:     ExpectBundleLength(2),
	},
}

// TestBundleFromFile goes through test cases defined in fileTests. See below for test cases definition and details.
func TestBundleFromFile(t *testing.T) {
	for _, test := range fileTests {
		b := newCustomizedBundlerFromFile(t, test.caBundleFile, test.intBundleFile, test.extraIntermediates)
		bundle, err := b.BundleFromFile(test.cert, test.key, Optimal, "")
		if test.errorCallback != nil {
			test.errorCallback(t, err)
		} else {
			if err != nil {
				t.Fatalf("expected no error bundling %q. but an error occurred: %v", test.cert, err)
			}
			if test.bundleChecking != nil {
				test.bundleChecking(t, bundle)
			}
		}

		if bundle != nil {
			bundle.Cert = nil
			if _, err = json.Marshal(bundle); err == nil {
				t.Fatal("bundle should fail with no cert")
			}
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Code generated by client-gen. DO NOT EDIT.

/*
Package applyconfigurations provides typesafe go representations of the apply
configurations that are used to constructs Server-side Apply requests.

Basics

The Apply functions in the typed client (see the k8s.io/client-go/kubernetes/typed packages) offer
a direct and typesafe way of calling Server-side Apply. Each Apply function takes an "apply
configuration" type as an argument, which is a structured representation of an Apply request. For
example:

    import (
         ...
         v1ac "k8s.io/client-go/applyconfigurations/autoscaling/v1"
    )
    hpaApplyConfig := v1ac.HorizontalPodAutoscaler(autoscalerName, ns).
         WithSpec(v1ac.HorizontalPodAutoscalerSpec().
                  WithMinReplicas(0)
         )
    return hpav1client.Apply(ctx, hpaApplyConfig, metav1.ApplyOptions{FieldManager: "mycontroller", Force: true})

Note in this example that HorizontalPodAutoscaler is imported from an "applyconfigurations"
package. Each "apply configuration" type represents the same Kubernetes object kind as the
corresponding go struct, but where all fields are pointers to make them optional, allowing apply
requests to be accurately represented. For example, this when the apply configuration in the above
example is marshalled to YAML, it produces:

    apiVersion: autoscaling/v1
    kind: HorizontalPodAutoscaler
    metadata:
        name: myHPA
        namespace: myNamespace
    spec:
        minReplicas: 0

To understand why this is needed, the above YAML cannot be produced by the
v1.HorizontalPodAutoscaler go struct. Take for example:

    hpa := v1.HorizontalPodAutoscaler{
         TypeMeta: metav1.TypeMeta{
                  APIVersion: "autoscaling/v1",
                  Kind:       "HorizontalPodAutoscaler",
         },
         ObjectMeta: ObjectMeta{
                  Namespace: ns,
                  Name:      autoscalerName,
         },
         Spec: v1.HorizontalPodAutoscalerSpec{
                  MinReplicas: pointer.Int32Ptr(0),
         },
    }

The above code attempts to declare the same apply configuration as shown in the previous examples,
but when marshalled to YAML, produces:

    kind: HorizontalPodAutoscaler
    apiVersion: autoscaling/v1
    metadata:
      name: myHPA
      namespace: myNamespace
      creationTimestamp: null
    spec:
      scaleTargetRef:
        kind: ""
        name: ""
      minReplicas: 0
      maxReplicas: 0

Which, among other things, contains spec.maxReplicas set to 0. This is almost certainly not what
the caller intended (the intended apply configuration says nothing about the maxReplicas field),
and could have serious consequences on a production system: it directs the autoscaler to downscale
to zero pods. The problem here originates from the fact that the go structs contain required fields
that are zero valued if not set explicitly. The go structs work as intended for create and update
operations, but are fundamentally incompatible with apply, which is why we have introduced the
generated "apply configuration" types.

The "apply configurations" also have convenience With<FieldName> functions that make it easier to
build apply requests. This allows developers to set fields without having to deal with the fact that
all the fields in the "apply configuration" types are pointers, and are inconvenient to set using
go. For example "MinReplicas: &0" is not legal go code, so without the With functions, developers
would work around this problem by using a library, .e.g. "MinReplicas: pointer.Int32Ptr(0)", but
string enumerations like corev1.Protocol are still a problem since they cannot be supported by a
general purpose library. In addition to the convenience, the With functions also isolate
developers from the underlying representation, which makes it safer for the underlying
representation to be changed to support additional features in the future.

Controller Support

The new client-go support makes it much easier to use Server-side Apply in controllers, by either of two mechanisms.

When authoring new controllers to use Server-side Apply, a good approach is to have the controller
recreate the apply configuration for an object each time it reconciles that object.  This ensures
that the controller fully reconciles all the fields that it is responsible for. Controllers
typically should unconditionally set all the fields they own by setting "Force: true" in the
ApplyOptions. Controllers must also provide a FieldManager name that is unique to the
reconciliation loop that apply is called from.

When upgrading existing controllers to use Server-side Apply the same approach often works
well--migrate the controllers to recreate the apply configuration each time it reconciles any
object. Unfortunately, the controller might have multiple code paths that update different parts of
an object depending on various conditions. Migrating a controller like this to Server-side Apply can
be risky because if the controller forgets to include any fields in an apply configuration that is
included in a previous apply request, a field can be accidentlly deleted. To ease this type of
migration, client-go apply support provides a way to replace any controller reconciliation code that
performs a "read/modify-in-place/update" (or patch) workflow with a "extract/modify-in-place/apply"
workflow. Here's an example of the new workflow:

    fieldMgr := "my-field-manager"
    deploymentClient := clientset.AppsV1().Deployments("default")
    // read, could also be read from a shared informer
    deployment, err := deploymentClient.Get(ctx, "example-deployment", metav1.GetOptions{})
    if err != nil {
      // handle error
    }
    // extract
    deploymentApplyConfig, err := appsv1ac.ExtractDeployment(deployment, fieldMgr)
    if err != nil {
      // handle error
    }
    // modify-in-place
    deploymentApplyConfig.Spec.Template.Spec.WithContainers(corev1ac.Container().
	WithName("modify-slice").
	WithImage("nginx:1.14.2"),
    )
    // apply
    applied, err := deploymentClient.Apply(ctx, extractedDeployment, metav1.ApplyOptions{FieldManager: fieldMgr})
*/
package applyconfigurations
<commit_msg>Split documentation according to both mechanisms available for migration<commit_after>/*
Copyright The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Code generated by applyconfiguration-gen. DO NOT EDIT.

/*
Package applyconfigurations provides typesafe go representations of the apply
configurations that are used to constructs Server-side Apply requests.

Basics

The Apply functions in the typed client (see the k8s.io/client-go/kubernetes/typed packages) offer
a direct and typesafe way of calling Server-side Apply. Each Apply function takes an "apply
configuration" type as an argument, which is a structured representation of an Apply request. For
example:

    import (
         ...
         v1ac "k8s.io/client-go/applyconfigurations/autoscaling/v1"
    )
    hpaApplyConfig := v1ac.HorizontalPodAutoscaler(autoscalerName, ns).
         WithSpec(v1ac.HorizontalPodAutoscalerSpec().
                  WithMinReplicas(0)
         )
    return hpav1client.Apply(ctx, hpaApplyConfig, metav1.ApplyOptions{FieldManager: "mycontroller", Force: true})

Note in this example that HorizontalPodAutoscaler is imported from an "applyconfigurations"
package. Each "apply configuration" type represents the same Kubernetes object kind as the
corresponding go struct, but where all fields are pointers to make them optional, allowing apply
requests to be accurately represented. For example, this when the apply configuration in the above
example is marshalled to YAML, it produces:

    apiVersion: autoscaling/v1
    kind: HorizontalPodAutoscaler
    metadata:
        name: myHPA
        namespace: myNamespace
    spec:
        minReplicas: 0

To understand why this is needed, the above YAML cannot be produced by the
v1.HorizontalPodAutoscaler go struct. Take for example:

    hpa := v1.HorizontalPodAutoscaler{
         TypeMeta: metav1.TypeMeta{
                  APIVersion: "autoscaling/v1",
                  Kind:       "HorizontalPodAutoscaler",
         },
         ObjectMeta: ObjectMeta{
                  Namespace: ns,
                  Name:      autoscalerName,
         },
         Spec: v1.HorizontalPodAutoscalerSpec{
                  MinReplicas: pointer.Int32Ptr(0),
         },
    }

The above code attempts to declare the same apply configuration as shown in the previous examples,
but when marshalled to YAML, produces:

    kind: HorizontalPodAutoscaler
    apiVersion: autoscaling/v1
    metadata:
      name: myHPA
      namespace: myNamespace
      creationTimestamp: null
    spec:
      scaleTargetRef:
        kind: ""
        name: ""
      minReplicas: 0
      maxReplicas: 0

Which, among other things, contains spec.maxReplicas set to 0. This is almost certainly not what
the caller intended (the intended apply configuration says nothing about the maxReplicas field),
and could have serious consequences on a production system: it directs the autoscaler to downscale
to zero pods. The problem here originates from the fact that the go structs contain required fields
that are zero valued if not set explicitly. The go structs work as intended for create and update
operations, but are fundamentally incompatible with apply, which is why we have introduced the
generated "apply configuration" types.

The "apply configurations" also have convenience With<FieldName> functions that make it easier to
build apply requests. This allows developers to set fields without having to deal with the fact that
all the fields in the "apply configuration" types are pointers, and are inconvenient to set using
go. For example "MinReplicas: &0" is not legal go code, so without the With functions, developers
would work around this problem by using a library, .e.g. "MinReplicas: pointer.Int32Ptr(0)", but
string enumerations like corev1.Protocol are still a problem since they cannot be supported by a
general purpose library. In addition to the convenience, the With functions also isolate
developers from the underlying representation, which makes it safer for the underlying
representation to be changed to support additional features in the future.

Controller Support

The new client-go support makes it much easier to use Server-side Apply in controllers, by either of
two mechanisms.

Mechanism 1:

When authoring new controllers to use Server-side Apply, a good approach is to have the controller
recreate the apply configuration for an object each time it reconciles that object.  This ensures
that the controller fully reconciles all the fields that it is responsible for. Controllers
typically should unconditionally set all the fields they own by setting "Force: true" in the
ApplyOptions. Controllers must also provide a FieldManager name that is unique to the
reconciliation loop that apply is called from.

When upgrading existing controllers to use Server-side Apply the same approach often works
well--migrate the controllers to recreate the apply configuration each time it reconciles any
object. For cases where this does not work well, see Mechanism 2.

Mechanism 2:

When upgrading existing controllers to use Server-side Apply, the controller might have multiple
code paths that update different parts of an object depending on various conditions. Migrating a
controller like this to Server-side Apply can be risky because if the controller forgets to include
any fields in an apply configuration that is included in a previous apply request, a field can be
accidentally deleted. For such cases, an alternative to mechanism 1 is to replace any controller
reconciliation code that performs a "read/modify-in-place/update" (or patch) workflow with a
"extract/modify-in-place/apply" workflow. Here's an example of the new workflow:

    fieldMgr := "my-field-manager"
    deploymentClient := clientset.AppsV1().Deployments("default")
    // read, could also be read from a shared informer
    deployment, err := deploymentClient.Get(ctx, "example-deployment", metav1.GetOptions{})
    if err != nil {
      // handle error
    }
    // extract
    deploymentApplyConfig, err := appsv1ac.ExtractDeployment(deployment, fieldMgr)
    if err != nil {
      // handle error
    }
    // modify-in-place
    deploymentApplyConfig.Spec.Template.Spec.WithContainers(corev1ac.Container().
	WithName("modify-slice").
	WithImage("nginx:1.14.2"),
    )
    // apply
    applied, err := deploymentClient.Apply(ctx, extractedDeployment, metav1.ApplyOptions{FieldManager: fieldMgr})
*/
package applyconfigurations
<|endoftext|>


<commit_before>package model

import (
	"fmt"
)

type Host struct {
	Id   int
	Name string
}

func (this *Host) String() string {
	return fmt.Sprintf(
		"<id:%s,name:%s>",
		this.Id,
		this.Name,
	)
}
<commit_msg>[IN-119] Fix wrong type error in coverage test.<commit_after>package model

import (
	"fmt"
)

type Host struct {
	Id   int
	Name string
}

func (this *Host) String() string {
	return fmt.Sprintf(
		"<id:%d,name:%s>",
		this.Id,
		this.Name,
	)
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"time"

	"github.com/apache/thrift/lib/go/thrift"
	"github.com/dt/go-metrics-reporting"
	"github.com/foursquare/gohfile"
	"github.com/foursquare/quiver/gen"
)

type HttpRpcHandler struct {
	stats *report.Recorder
	*gen.HFileServiceProcessor
}

func NewHttpRpcHandler(cs *hfile.CollectionSet, stats *report.Recorder) *HttpRpcHandler {
	impl := gen.NewHFileServiceProcessor(&ThriftRpcImpl{cs})
	return &HttpRpcHandler{stats, impl}
}

// borrowed from generated thrift code, but with instrumentation added.
func (p *HttpRpcHandler) Process(iprot, oprot thrift.TProtocol) (success bool, err thrift.TException) {
	name, _, seqId, err := iprot.ReadMessageBegin()
	if err != nil {
		return false, err
	}

	if processor, ok := p.GetProcessorFunction(name); ok {
		start := time.Now()
		success, err = processor.Process(seqId, iprot, oprot)
		if p.stats != nil {
			p.stats.TimeSince(name, start)
		}
		return
	}

	iprot.Skip(thrift.STRUCT)
	iprot.ReadMessageEnd()
	e := thrift.NewTApplicationException(thrift.UNKNOWN_METHOD, "Unknown function "+name)

	oprot.WriteMessageBegin(name, thrift.EXCEPTION, seqId)
	e.Write(oprot)
	oprot.WriteMessageEnd()
	oprot.Flush()

	return false, e
}

func (h *HttpRpcHandler) ServeHTTP(out http.ResponseWriter, req *http.Request) {
	if req.Method == "POST" {
		var in *thrift.TMemoryBuffer
		size := int(req.ContentLength)
		if size > 0 {
			in = thrift.NewTMemoryBufferLen(size)
		} else {
			in = thrift.NewTMemoryBuffer()
		}

		in.ReadFrom(req.Body)
		defer req.Body.Close()

		compact := false

		if in.Len() > 0 && in.Bytes()[0] == thrift.COMPACT_PROTOCOL_ID {
			compact = true
		}

		outbuf := thrift.NewTMemoryBuffer()

		var iprot thrift.TProtocol
		var oprot thrift.TProtocol

		if compact {
			iprot = thrift.NewTCompactProtocol(in)
			oprot = thrift.NewTCompactProtocol(outbuf)
		} else {
			iprot = thrift.NewTBinaryProtocol(in, true, true)
			oprot = thrift.NewTBinaryProtocol(outbuf, true, true)
		}

		ok, err := h.Process(iprot, oprot)

		if ok {
			outbuf.WriteTo(out)
		} else {
			http.Error(out, err.Error(), 500)
		}
	} else {
		http.Error(out, "Must POST TBinary encoded thrift RPC", 401)
	}
}
<commit_msg>arena alloc thrift buffers<commit_after>package main

import (
	"net/http"
	"sync"
	"time"

	"github.com/apache/thrift/lib/go/thrift"
	"github.com/dt/go-metrics-reporting"
	"github.com/foursquare/gohfile"
	"github.com/foursquare/quiver/gen"
)

type HttpRpcHandler struct {
	stats   *report.Recorder
	buffers sync.Pool
	*gen.HFileServiceProcessor
}

func NewHttpRpcHandler(cs *hfile.CollectionSet, stats *report.Recorder) *HttpRpcHandler {
	impl := gen.NewHFileServiceProcessor(&ThriftRpcImpl{cs})
	return &HttpRpcHandler{stats, sync.Pool{}, impl}
}

// borrowed from generated thrift code, but with instrumentation added.
func (p *HttpRpcHandler) Process(iprot, oprot thrift.TProtocol) (success bool, err thrift.TException) {
	name, _, seqId, err := iprot.ReadMessageBegin()
	if err != nil {
		return false, err
	}

	if processor, ok := p.GetProcessorFunction(name); ok {
		start := time.Now()
		success, err = processor.Process(seqId, iprot, oprot)
		if p.stats != nil {
			p.stats.TimeSince(name, start)
		}
		return
	}

	iprot.Skip(thrift.STRUCT)
	iprot.ReadMessageEnd()
	e := thrift.NewTApplicationException(thrift.UNKNOWN_METHOD, "Unknown function "+name)

	oprot.WriteMessageBegin(name, thrift.EXCEPTION, seqId)
	e.Write(oprot)
	oprot.WriteMessageEnd()
	oprot.Flush()

	return false, e
}

func (h *HttpRpcHandler) getBuf() *thrift.TMemoryBuffer {
	res := h.buffers.Get()
	if res == nil {
		return thrift.NewTMemoryBuffer()
	} else {
		out := res.(*thrift.TMemoryBuffer)
		out.Reset()
		return out
	}
}

func (h *HttpRpcHandler) ServeHTTP(out http.ResponseWriter, req *http.Request) {
	if req.Method == "POST" {
		in := h.getBuf()
		defer h.buffers.Put(in)

		in.ReadFrom(req.Body)
		defer req.Body.Close()

		compact := false

		if in.Len() > 0 && in.Bytes()[0] == thrift.COMPACT_PROTOCOL_ID {
			compact = true
		}

		outbuf := h.getBuf()
		defer h.buffers.Put(outbuf)

		var iprot thrift.TProtocol
		var oprot thrift.TProtocol

		if compact {
			iprot = thrift.NewTCompactProtocol(in)
			oprot = thrift.NewTCompactProtocol(outbuf)
		} else {
			iprot = thrift.NewTBinaryProtocol(in, true, true)
			oprot = thrift.NewTBinaryProtocol(outbuf, true, true)
		}

		ok, err := h.Process(iprot, oprot)

		if ok {
			outbuf.WriteTo(out)
		} else {
			http.Error(out, err.Error(), 500)
		}
	} else {
		http.Error(out, "Must POST TBinary encoded thrift RPC", 401)
	}
}
<|endoftext|>


<commit_before>package httpclient
import (
	"net/http"
	"log"
	"errors"
	"os"
)
func CheckMultipart(urls string) (bool, error) {
	r, err := http.NewRequest("GET", urls, nil)
	if err!=nil {
		return false, err
	}
	r.Header.Add("Range", "bytes=0-0")
	cl := http.Client{}
	f, _ := os.Create("/home/andrew/Desktop/dum.txt")
	r.Write(f)
	defer f.Close()
	resp, err := cl.Do(r)
	if err!=nil {
		log.Printf("error: can't check multipart support assume no %v \n", err)
		return false, err
	}
	f1, _ := os.Create("/home/andrew/Desktop/res.txt")
	resp.Write(f1)
	if resp.StatusCode!=206 {
		return false, errors.New("error: file not found or moved status: "+ resp.Status)
	}
	if (resp.ContentLength==1) {
		log.Printf("info: file size is %d bytes \n", resp.ContentLength)
		return true, nil
	}
	return false, nil
}

func GetSize(urls string) (int64, error) {
	cl := http.Client{}
	resp, err := cl.Head(urls)
	if err!=nil {
		log.Printf("error: when try get file size %v \n", err)
		return 0, err
	}
	if resp.StatusCode!=200 {
		log.Printf("error: file not found or moved status:", resp.StatusCode)
		return 0, errors.New("error: file not found or moved")
	}
	log.Printf("info: file size is %d bytes \n", resp.ContentLength)
	return resp.ContentLength, nil
}

type PartialDownloader struct {
	from int64
	to   int64
	pos  int64
	rch  bool
	url  string
}
func (pd *PartialDownloader) Init(url string, from int64, pos int64, to int64) {
	pd.from=from
	pd.to=to
	pd.pos=pos
}
func (pd PartialDownloader) GetProgress() interface{} {
	return pd.from//, pd.to, pd.to
}
func (pd *PartialDownloader) DoWork() (bool, error) {
	//in last time we check resume support
	if !pd.rch {
		if nos, err := CheckMultipart(pd.url); nos {
			return false, err
		}
	}
	//assume resume support
	pd.rch=true
	//do download

	return true, nil
}
<commit_msg>prepare download method<commit_after>package httpclient
import (
	"net/http"
	"log"
	"errors"
	"os"
)
const MaxDownloadPortion=4096
func CheckMultipart(urls string) (bool, error) {
	r, err := http.NewRequest("GET", urls, nil)
	if err!=nil {
		return false, err
	}
	r.Header.Add("Range", "bytes=0-0")
	cl := http.Client{}
	resp, err := cl.Do(r)
	if err!=nil {
		log.Printf("error: can't check multipart support assume no %v \n", err)
		return false, err
	}
	f1, _ := os.Create("/home/andrew/Desktop/res.txt")
	resp.Write(f1)
	if resp.StatusCode!=206 {
		return false, errors.New("error: file not found or moved status: "+ resp.Status)
	}
	if (resp.ContentLength==1) {
		log.Printf("info: file size is %d bytes \n", resp.ContentLength)
		return true, nil
	}
	return false, nil
}

func GetSize(urls string) (int64, error) {
	cl := http.Client{}
	resp, err := cl.Head(urls)
	if err!=nil {
		log.Printf("error: when try get file size %v \n", err)
		return 0, err
	}
	if resp.StatusCode!=200 {
		log.Printf("error: file not found or moved status:", resp.StatusCode)
		return 0, errors.New("error: file not found or moved")
	}
	log.Printf("info: file size is %d bytes \n", resp.ContentLength)
	return resp.ContentLength, nil
}

type DownloadProgress struct{
	from int64
	to   int64
	pos  int64
}
type PartialDownloader struct {
	dp DownloadProgress
	cli *http.Client
	rch  bool
	url  string
}
func (pd *PartialDownloader) Init(url string, from int64, pos int64, to int64) {
	pd.dp.from=from
	pd.dp.to=to
	pd.dp.pos=pos
}

func (pd PartialDownloader) GetProgress()DownloadProgress{
	return pd.dp
}
func constuctReqH(current int64,to int64)string{
	if to<current+MaxDownloadPortion{
		return "bytes="+current+"-"+to
	}

	return "bytes="+current+"-"+MaxDownloadPortion

}
func (pd *PartialDownloader) DoWork() (bool, error) {
	//in last time we check resume support
	if !pd.rch {
		if nos, err := CheckMultipart(pd.url); nos {
			return false, err
		}
	}
	//assume resume support
	pd.rch=true
	//do download

	//check if our client is not created
	if pd.cli==nil{
		pd.cli=new(http.Client)
	}
	//create new req
	r, err := http.NewRequest("GET", pd.url, nil)
	//ok we construct query
	r.Header.Add("Range", constuctReqH(pd.dp.pos,pd.dp.to))
	if err!=nil {
		return false, err
	}
	//try send
	resp, err := pd.cli.Do(r)
	if err!=nil {
		log.Printf("error: error download part  file%v \n", err)
		return false, err
	}
	//check response
	if resp.StatusCode!=200 {
		log.Printf("error: file not found or moved status:", resp.StatusCode)
		return false, errors.New("error: file not found or moved")
	}
	pd.dp.pos+pd.dp.pos+MaxDownloadPortion
	if(pd.dp.pos==pd.dp.to){
		//ok download part complete normal
		return false,nil
	}
	//not full download try next segment
	return true, nil
}
<|endoftext|>


<commit_before>package ogo

import (
	"fmt"
	"mime"
	"net/http"
	//"path/filepath"
	"runtime/debug"
	"strings"
	"time"

	"github.com/Odinman/ogo/utils"
	"github.com/dustin/randbo"
	"github.com/zenazn/goji/web"
)

// Key to use when setting the request ID.
const (
	//固定参数名称
	_PARAM_FIELDS  = "fields"
	_PARAM_PAGE    = "page"
	_PARAM_PERPAGE = "per_page"
	_PARAM_DATE    = "date"
	_PARAM_START   = "start"
	_PARAM_END     = "end"
	_PARAM_ORDERBY = "orderby"

	//特殊前缀
	_PPREFIX_NOT  = '!'
	_PPREFIX_LIKE = '~'

	// 查询类型
	_CTYPE_IS   = 0
	_CTYPE_NOT  = 1
	_CTYPE_LIKE = 2
	_CTYPE_JOIN = 3

	OriginalRemoteAddrKey = "originalRemoteAddr"
)

var (
	xForwardedFor      = http.CanonicalHeaderKey("X-Forwarded-For")
	xRealIP            = http.CanonicalHeaderKey("X-Real-IP")
	contentType        = http.CanonicalHeaderKey("Content-Type")
	contentDisposition = http.CanonicalHeaderKey("Content-Disposition")
	contentMD5         = http.CanonicalHeaderKey("Content-MD5")
	rcHolder           func(c web.C, w http.ResponseWriter, r *http.Request) *RESTContext
)

/* {{{ func EnvInit(c *web.C, h http.Handler) http.Handler
 * 初始化环境
 */
func EnvInit(c *web.C, h http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		ac := new(Access) //access日志信息
		ac.Time = time.Now()
		ac.Method = r.Method
		ac.URI = r.RequestURI
		ac.Proto = r.Proto
		ac.Host = r.Host
		ac.InHeader = &r.Header
		// env
		if c.Env == nil {
			c.Env = make(map[string]interface{})
		}

		// make rand string(for debug, session...)
		buf := make([]byte, 16)
		randbo.New().Read(buf) //号称最快的随机字符串
		ac.Session = fmt.Sprintf("%x", buf)

		c.Env[RequestIDKey] = ac.Session

		c.Env[LogPrefixKey] = "[" + ac.Session[:10] + "]" //只显示前十位

		Debug("[%s] [%s %s] started", ac.Session[:10], r.Method, r.RequestURI)

		lw := utils.WrapWriter(w)

		pathPieces := strings.Split(r.URL.Path, "/")
		for off, piece := range pathPieces {
			if piece != "" {
				if off == 1 {
					c.Env[EndpointKey] = piece
				}
				if off == 2 && piece[0] != '@' { //@开头是selector
					c.Env[RowkeyKey] = piece
				}
				if off > 1 && piece[0] == '@' {
					c.Env[SelectorKey] = piece
				}
			}
		}
		// real ip(处理在代理服务器之后的情况)
		if rip := realIP(r); rip != "" {
			c.Env[OriginalRemoteAddrKey] = r.RemoteAddr
			r.RemoteAddr = rip
		}
		ac.IP = r.RemoteAddr

		//init RESTContext
		var rcErr error
		var rc *RESTContext
		rc, rcHolder, rcErr = RCHolder(*c, w, r)
		if na, ok := rc.Route.Options[NoLogKey]; ok && na.(bool) == true {
			rc.SetEnv(NoLogKey, true)
		}
		rc.Access = ac
		rc.Access.ReqLength = len(rc.RequestBody)
		if rcErr != nil {
			rc.RESTBadRequest(rcErr)
			return
		}

		h.ServeHTTP(lw, r)

	}

	return http.HandlerFunc(fn)
}

/* }}} */

/* {{{ func Defer(c *web.C, h http.Handler) http.Handler
 * recovers from panics
 */
func Defer(c *web.C, h http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {

		rc := rcHolder(*c, w, r)
		//Debug("defer len: %d", len(rc.RequestBody))
		defer func() {
			if err := recover(); err != nil {
				rc.Critical("[%s %s] %v", r.Method, r.URL.Path, err)
				//debug.PrintStack()
				rc.Critical("%s", debug.Stack())
				//http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
				rc.HTTPError(http.StatusInternalServerError)
			}

			// save access log here
			ac := rc.Access
			ac.Duration = time.Now().Sub(ac.Time).String()
			ac.Status = rc.Status
			ac.OutHeader = w.Header()
			ac.RepLength = rc.ContentLength
			if sb := rc.GetEnv(SaveBodyKey); sb != nil && sb.(bool) == true {
				//可以由应用程序决定是否记录body
				ac.ReqBody = string(rc.RequestBody)
			}
			//ac.App = string(rc.RequestBody)

			Debug("[%s] [%s %s] end:%d in %s", ac.Session[:10], ac.Method, ac.URI, ac.Status, ac.Duration)
			// save access
			rc.SaveAccess()
		}()

		h.ServeHTTP(w, r)
	}

	return http.HandlerFunc(fn)
}

/* }}} */

/* {{{ func Mime(c *web.C, h http.Handler) http.Handler
 * mimetype相关处理
 */
func Mime(c *web.C, h http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {

		rc := rcHolder(*c, w, r)

		if cs := r.Header.Get(contentMD5); cs != "" {
			rc.SetEnv(ContentMD5Key, cs)
		}

		// 看content-type
		if ct := r.Header.Get(contentType); ct != "" {
			rc.SetEnv(MimeTypeKey, ct)
		}
		if cd := r.Header.Get(contentDisposition); cd != "" {
			//以传入的Disposition为主
			if t, m, e := mime.ParseMediaType(cd); e == nil {
				rc.Info("disposition: %s, mediatype: %s", cd, t)
				rc.SetEnv(DispositionMTKey, t)
				//if fname, ok := m["filename"]; ok {
				//	if mt := mime.TypeByExtension(filepath.Ext(fname)); mt != "" {
				//		rc.SetEnv(MimeTypeKey, mt)
				//	}
				//}
				for k, v := range m {
					dk := DispositionPrefix + k + "_"
					rc.Debug("disposition key: %s, value: %v", dk, v)
					rc.SetEnv(dk, v)
				}
			}

		}

		h.ServeHTTP(w, r)
	}

	return http.HandlerFunc(fn)
}

/* }}} */

/* {{{ func ParseParams(c *web.C, h http.Handler) http.Handler {
 *
 */
func ParseParams(c *web.C, h http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {

		rc := rcHolder(*c, w, r)
		// 解析参数
		r.ParseForm()
		// 根据ogo规则解析参数
		var cType int
		var p, pp string
		rc.setTimeRangeFromStartEnd()
		for k, v := range r.Form {
			rc.Trace("key: %s, value: %s", k, v)
			//根据参数名第一个字符来判断条件类型
			prefix := k[0] //param prefix
			switch prefix {
			case _PPREFIX_NOT:
				rc.Trace("having prefix not: %s", k)
				k = k[1:]
				cType = _CTYPE_NOT
				rc.Trace("key change to: %s, condition type: %d", k, cType)
			case _PPREFIX_LIKE:
				k = k[1:]
				cType = _CTYPE_LIKE
			default:
				cType = _CTYPE_IS
				// do nothing
			}

			switch k { //处理参数
			case _PARAM_DATE:
				rc.setTimeRangeFromDate(v)
			case _PARAM_ORDERBY:
				rc.setOrderBy(v)
			case _PARAM_FIELDS:
				//过滤字段
				rc.SetEnv(FieldsKey, v)
			case _PARAM_PERPAGE:
				if len(v) > 0 {
					pp = v[0]
				}
			case _PARAM_PAGE: //分页信息
				if len(v) > 0 {
					p = v[0]
				}
			default:
				//除了以上的特别字段,其他都是条件查询
				var cv interface{}
				var con, jc *Condition
				var err error

				if len(v) > 1 {
					cv = v
				} else {
					//cv = v[0]
					//处理逗号情况
					if strings.Contains(v[0], ",") {
						cv = strings.Split(v[0], ",")
					} else {
						cv = v[0]
					}
				}

				//如果参数中包含".",代表有关联查询
				if strings.Contains(k, ".") {
					js := strings.SplitN(k, ".", 2)
					if js[0] != "" && js[1] != "" {
						k = js[0]
						jc = new(Condition)
						jc.Field = js[1]
						switch cType {
						case _CTYPE_IS:
							jc.Is = cv
						case _CTYPE_NOT:
							jc.Not = cv
						case _CTYPE_LIKE:
							jc.Like = cv
						default:
						}
						//查询类型变为join
						rc.Trace("join: %s, %s; con: %v", k, jc.Field, jc)
						cType = _CTYPE_JOIN
					}
				}

				if con, err = rc.GetCondition(k); err != nil {
					//没有这个condition,初始化
					con = new(Condition)
					rc.setCondition(k, con)
				}
				con.Field = k
				switch cType {
				case _CTYPE_IS:
					con.Is = cv
				case _CTYPE_NOT:
					con.Not = cv
				case _CTYPE_LIKE:
					con.Like = cv
				case _CTYPE_JOIN:
					rc.Trace("field: %s, join condition: %v", k, jc)
					con.Join = jc
				default:
				}
				rc.Trace("con: %v", con)
			}
		}
		//记录分页信息
		rc.SetEnv(PaginationKey, NewPagination(p, pp))

		h.ServeHTTP(w, r)
	}

	return http.HandlerFunc(fn)
}

/* }}} */

/* {{{ func realIP(r *http.Request) string
 * 获取真实IP
 */
func realIP(r *http.Request) string {
	var ip string

	if xff := r.Header.Get(xForwardedFor); xff != "" {
		i := strings.Index(xff, ", ")
		if i == -1 {
			i = len(xff)
		}
		ip = xff[:i]
	} else if xrip := r.Header.Get(xRealIP); xrip != "" {
		ip = xrip
	}

	return ip
}

/* }}} */
<commit_msg>update<commit_after>package ogo

import (
	"fmt"
	"mime"
	"net/http"
	//"path/filepath"
	"runtime/debug"
	"strings"
	"time"

	"github.com/Odinman/ogo/utils"
	"github.com/dustin/randbo"
	"github.com/zenazn/goji/web"
)

// Key to use when setting the request ID.
const (
	//固定参数名称
	_PARAM_FIELDS  = "fields"
	_PARAM_PAGE    = "page"
	_PARAM_PERPAGE = "per_page"
	_PARAM_DATE    = "date"
	_PARAM_START   = "start"
	_PARAM_END     = "end"
	_PARAM_ORDERBY = "orderby"

	//特殊前缀
	_PPREFIX_NOT  = '!'
	_PPREFIX_LIKE = '~'

	// 查询类型
	_CTYPE_IS   = 0
	_CTYPE_NOT  = 1
	_CTYPE_LIKE = 2
	_CTYPE_JOIN = 3

	OriginalRemoteAddrKey = "originalRemoteAddr"
)

var (
	xForwardedFor      = http.CanonicalHeaderKey("X-Forwarded-For")
	xRealIP            = http.CanonicalHeaderKey("X-Real-IP")
	contentType        = http.CanonicalHeaderKey("Content-Type")
	contentDisposition = http.CanonicalHeaderKey("Content-Disposition")
	contentMD5         = http.CanonicalHeaderKey("Content-MD5")
	rcHolder           func(c web.C, w http.ResponseWriter, r *http.Request) *RESTContext
)

/* {{{ func EnvInit(c *web.C, h http.Handler) http.Handler
 * 初始化环境
 */
func EnvInit(c *web.C, h http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		ac := new(Access) //access日志信息
		ac.Time = time.Now()
		ac.Method = r.Method
		ac.URI = r.RequestURI
		ac.Proto = r.Proto
		ac.Host = r.Host
		ac.InHeader = &r.Header
		// env
		if c.Env == nil {
			c.Env = make(map[string]interface{})
		}

		// make rand string(for debug, session...)
		buf := make([]byte, 16)
		randbo.New().Read(buf) //号称最快的随机字符串
		ac.Session = fmt.Sprintf("%x", buf)

		c.Env[RequestIDKey] = ac.Session

		c.Env[LogPrefixKey] = "[" + ac.Session[:10] + "]" //只显示前十位

		Trace("[%s] [%s %s] started", ac.Session[:10], r.Method, r.RequestURI)

		lw := utils.WrapWriter(w)

		pathPieces := strings.Split(r.URL.Path, "/")
		for off, piece := range pathPieces {
			if piece != "" {
				if off == 1 {
					c.Env[EndpointKey] = piece
				}
				if off == 2 && piece[0] != '@' { //@开头是selector
					c.Env[RowkeyKey] = piece
				}
				if off > 1 && piece[0] == '@' {
					c.Env[SelectorKey] = piece
				}
			}
		}
		// real ip(处理在代理服务器之后的情况)
		if rip := realIP(r); rip != "" {
			c.Env[OriginalRemoteAddrKey] = r.RemoteAddr
			r.RemoteAddr = rip
		}
		ac.IP = r.RemoteAddr

		//init RESTContext
		var rcErr error
		var rc *RESTContext
		rc, rcHolder, rcErr = RCHolder(*c, w, r)
		if na, ok := rc.Route.Options[NoLogKey]; ok && na.(bool) == true {
			rc.SetEnv(NoLogKey, true)
		}
		rc.Access = ac
		rc.Access.ReqLength = len(rc.RequestBody)
		if rcErr != nil {
			rc.RESTBadRequest(rcErr)
			return
		}

		h.ServeHTTP(lw, r)

	}

	return http.HandlerFunc(fn)
}

/* }}} */

/* {{{ func Defer(c *web.C, h http.Handler) http.Handler
 * recovers from panics
 */
func Defer(c *web.C, h http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {

		rc := rcHolder(*c, w, r)
		defer func() {
			if err := recover(); err != nil {
				rc.Critical("[%s %s] %v", r.Method, r.URL.Path, err)
				//debug.PrintStack()
				rc.Critical("%s", debug.Stack())
				//http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
				rc.HTTPError(http.StatusInternalServerError)
			}

			// save access log here
			ac := rc.Access
			ac.Duration = time.Now().Sub(ac.Time).String()
			ac.Status = rc.Status
			ac.OutHeader = w.Header()
			ac.RepLength = rc.ContentLength
			if sb := rc.GetEnv(SaveBodyKey); sb != nil && sb.(bool) == true {
				//可以由应用程序决定是否记录body
				ac.ReqBody = string(rc.RequestBody)
			}
			//ac.App = string(rc.RequestBody)

			rc.Debug("[%s %s] end:%d in %s", ac.Method, ac.URI, ac.Status, ac.Duration)
			// save access
			rc.SaveAccess()
		}()

		h.ServeHTTP(w, r)
	}

	return http.HandlerFunc(fn)
}

/* }}} */

/* {{{ func Mime(c *web.C, h http.Handler) http.Handler
 * mimetype相关处理
 */
func Mime(c *web.C, h http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {

		rc := rcHolder(*c, w, r)

		if cs := r.Header.Get(contentMD5); cs != "" {
			rc.SetEnv(ContentMD5Key, cs)
		}

		// 看content-type
		if ct := r.Header.Get(contentType); ct != "" {
			rc.SetEnv(MimeTypeKey, ct)
		}
		if cd := r.Header.Get(contentDisposition); cd != "" {
			//以传入的Disposition为主
			if t, m, e := mime.ParseMediaType(cd); e == nil {
				rc.Info("disposition: %s, mediatype: %s", cd, t)
				rc.SetEnv(DispositionMTKey, t)
				//if fname, ok := m["filename"]; ok {
				//	if mt := mime.TypeByExtension(filepath.Ext(fname)); mt != "" {
				//		rc.SetEnv(MimeTypeKey, mt)
				//	}
				//}
				for k, v := range m {
					dk := DispositionPrefix + k + "_"
					rc.Debug("disposition key: %s, value: %v", dk, v)
					rc.SetEnv(dk, v)
				}
			}

		}

		h.ServeHTTP(w, r)
	}

	return http.HandlerFunc(fn)
}

/* }}} */

/* {{{ func ParseParams(c *web.C, h http.Handler) http.Handler {
 *
 */
func ParseParams(c *web.C, h http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {

		rc := rcHolder(*c, w, r)
		// 解析参数
		r.ParseForm()
		// 根据ogo规则解析参数
		var cType int
		var p, pp string
		rc.setTimeRangeFromStartEnd()
		for k, v := range r.Form {
			rc.Trace("key: %s, value: %s", k, v)
			//根据参数名第一个字符来判断条件类型
			prefix := k[0] //param prefix
			switch prefix {
			case _PPREFIX_NOT:
				rc.Trace("having prefix not: %s", k)
				k = k[1:]
				cType = _CTYPE_NOT
				rc.Trace("key change to: %s, condition type: %d", k, cType)
			case _PPREFIX_LIKE:
				k = k[1:]
				cType = _CTYPE_LIKE
			default:
				cType = _CTYPE_IS
				// do nothing
			}

			switch k { //处理参数
			case _PARAM_DATE:
				rc.setTimeRangeFromDate(v)
			case _PARAM_ORDERBY:
				rc.setOrderBy(v)
			case _PARAM_FIELDS:
				//过滤字段
				rc.SetEnv(FieldsKey, v)
			case _PARAM_PERPAGE:
				if len(v) > 0 {
					pp = v[0]
				}
			case _PARAM_PAGE: //分页信息
				if len(v) > 0 {
					p = v[0]
				}
			default:
				//除了以上的特别字段,其他都是条件查询
				var cv interface{}
				var con, jc *Condition
				var err error

				if len(v) > 1 {
					cv = v
				} else {
					//cv = v[0]
					//处理逗号情况
					if strings.Contains(v[0], ",") {
						cv = strings.Split(v[0], ",")
					} else {
						cv = v[0]
					}
				}

				//如果参数中包含".",代表有关联查询
				if strings.Contains(k, ".") {
					js := strings.SplitN(k, ".", 2)
					if js[0] != "" && js[1] != "" {
						k = js[0]
						jc = new(Condition)
						jc.Field = js[1]
						switch cType {
						case _CTYPE_IS:
							jc.Is = cv
						case _CTYPE_NOT:
							jc.Not = cv
						case _CTYPE_LIKE:
							jc.Like = cv
						default:
						}
						//查询类型变为join
						rc.Trace("join: %s, %s; con: %v", k, jc.Field, jc)
						cType = _CTYPE_JOIN
					}
				}

				if con, err = rc.GetCondition(k); err != nil {
					//没有这个condition,初始化
					con = new(Condition)
					rc.setCondition(k, con)
				}
				con.Field = k
				switch cType {
				case _CTYPE_IS:
					con.Is = cv
				case _CTYPE_NOT:
					con.Not = cv
				case _CTYPE_LIKE:
					con.Like = cv
				case _CTYPE_JOIN:
					rc.Trace("field: %s, join condition: %v", k, jc)
					con.Join = jc
				default:
				}
				rc.Trace("con: %v", con)
			}
		}
		//记录分页信息
		rc.SetEnv(PaginationKey, NewPagination(p, pp))

		h.ServeHTTP(w, r)
	}

	return http.HandlerFunc(fn)
}

/* }}} */

/* {{{ func realIP(r *http.Request) string
 * 获取真实IP
 */
func realIP(r *http.Request) string {
	var ip string

	if xff := r.Header.Get(xForwardedFor); xff != "" {
		i := strings.Index(xff, ", ")
		if i == -1 {
			i = len(xff)
		}
		ip = xff[:i]
	} else if xrip := r.Header.Get(xRealIP); xrip != "" {
		ip = xrip
	}

	return ip
}

/* }}} */
<|endoftext|>


<commit_before>// Copyright 2015 - 2016 Square Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package filter

import (
	"math"
	"sort"
	"time"

	"github.com/square/metrics/api"
)

type filterList struct {
	index     []int
	value     []float64
	ascending bool
}

func (list filterList) Len() int {
	return len(list.index)
}
func (list filterList) Less(i, j int) bool {
	if math.IsNaN(list.value[j]) && !math.IsNaN(list.value[i]) {
		return true
	}
	if list.ascending {
		return list.value[i] < list.value[j]
	} else {
		return list.value[j] < list.value[i]
	}

}
func (list filterList) Swap(i, j int) {
	list.index[i], list.index[j] = list.index[j], list.index[i]
	list.value[i], list.value[j] = list.value[j], list.value[i]
}

func newFilterList(size int, ascending bool) filterList {
	return filterList{
		index:     make([]int, size),
		value:     make([]float64, size),
		ascending: ascending,
	}
}

// FilterBy reduces the number of things in the series `list` to at most the given `count`.
// They're chosen by sorting by `summary` in `ascending` or descending order.
func FilterBy(list api.SeriesList, count int, summary func([]float64) float64, lowest bool) api.SeriesList {
	if len(list.Series) < count {
		// Limit the count to the number of available series
		count = len(list.Series)
	}
	array := newFilterList(len(list.Series), lowest)
	for i := range array.index {
		array.index[i] = i
		array.value[i] = summary(list.Series[i].Values)
	}
	sort.Sort(array)

	series := make([]api.Timeseries, count)
	for i := range series {
		series[i] = list.Series[array.index[i]]
	}

	return api.SeriesList{
		Series:    series,
		Timerange: list.Timerange,
	}
}

func FilterRecentBy(list api.SeriesList, count int, summary func([]float64) float64, lowest bool, duration time.Duration) api.SeriesList {
	if len(list.Series) < count {
		count = len(list.Series) // Limit the count to the number of available series
	}
	array := newFilterList(len(list.Series), lowest)

	if list.Timerange.Resolution() == 0 {
		panic("FilterRecentBy received a api.SeriesList without a Resolution.")
	}

	// The number of elements to include
	elements := int(duration / list.Timerange.Resolution())
	if elements < 1 {
		elements = 1
	}
	if elements > list.Timerange.Slots() {
		elements = list.Timerange.Slots()
	}
	for i := range array.index {
		array.index[i] = i
		values := list.Series[i].Values
		// Include only the last `elements`.
		array.value[i] = summary(values[len(values)-elements:])
	}
	sort.Sort(array)

	series := make([]api.Timeseries, count)
	for i := range series {
		series[i] = list.Series[array.index[i]]
	}

	return api.SeriesList{
		Series:    series,
		Timerange: list.Timerange,
	}
}
<commit_msg>define filter threshold<commit_after>// Copyright 2015 - 2016 Square Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package filter

import (
	"math"
	"sort"
	"time"

	"github.com/square/metrics/api"
)

type filterList struct {
	index     []int
	value     []float64
	ascending bool
}

func (list filterList) Len() int {
	return len(list.index)
}
func (list filterList) Less(i, j int) bool {
	if math.IsNaN(list.value[j]) && !math.IsNaN(list.value[i]) {
		return true
	}
	if list.ascending {
		return list.value[i] < list.value[j]
	} else {
		return list.value[j] < list.value[i]
	}

}
func (list filterList) Swap(i, j int) {
	list.index[i], list.index[j] = list.index[j], list.index[i]
	list.value[i], list.value[j] = list.value[j], list.value[i]
}

func newFilterList(size int, ascending bool) filterList {
	return filterList{
		index:     make([]int, size),
		value:     make([]float64, size),
		ascending: ascending,
	}
}

func sortSeries(series []api.Timeseries, summary func([]float64) float64, lowest bool) ([]api.Timeseries, []float64) {
	array := newFilterList(len(series), lowest)
	for i := range array.index {
		array.index[i] = i
		array.value[i] = summary(series[i].Values)
	}
	sort.Sort(array)
	result := make([]api.Timeseries, len(series))
	weights := make([]float64, len(series))
	for i, index := range array.index {
		result[i] = series[index]
		weights[i] = array.value[index]
	}
	return result, weights
}

// FilterBy reduces the number of things in the series `list` to at most the given `count`.
// They're chosen by sorting by `summary` in `ascending` or descending order.
func FilterBy(list api.SeriesList, count int, summary func([]float64) float64, lowest bool) api.SeriesList {
	sorted, _ := sortSeries(list.Series, summary, lowest)

	if len(list.Series) < count {
		// Limit the count to the number of available series
		count = len(list.Series)
	}

	return api.SeriesList{
		Series:    sorted[:count],
		Timerange: list.Timerange,
	}
}

// FilterRecentBy reduces the number of things in the series `list` to at most the given `count`.
// However, it only considered recent points when evaluating their ordering.
func FilterRecentBy(list api.SeriesList, count int, summary func([]float64) float64, lowest bool, duration time.Duration) api.SeriesList {
	slots := int(duration / list.Timerange.Resolution())
	if slots <= 0 {
		slots = 1
	}
	if slots > list.Timerange.Slots() {
		slots = list.Timerange.Slots()
	}
	sorted, _ := sortSeries(list.Series, func(values []float64) float64 {
		return summary(values[len(values)-slots:])
	}, lowest)

	if len(list.Series) < count {
		// Limit the count to the number of available series
		count = len(list.Series)
	}

	return api.SeriesList{
		Series:    sorted[:count],
		Timerange: list.Timerange,
	}
}

// FilterThresholdBy reduces the number of things in the series `list` to those whose `summar` is at at least/at most the threshold.
func FilterThresholdBy(list api.SeriesList, threshold float64, summary func([]float64) float64, lowest bool) api.SeriesList {
	sorted, values := sortSeries(list.Series, summary, lowest)

	result := []api.Timeseries{}
	for i := range sorted {

		// Since the series are sorted, once one of them falls outside the threshold, we can stop.
		if (lowest && values[i] > threshold) || (!lowest && values[i] < threshold) {
			break
		}
		result = append(result, sorted[i])
	}

	return api.SeriesList{
		Series:    result,
		Timerange: list.Timerange,
	}
}

// FilterThresholdBy reduces the number of things in the series `list` to those whose `summar` is at at least/at most the threshold.
// However, it only considers the data points as recent as the duration permits.
func FilterThresholdRecentBy(list api.SeriesList, threshold float64, summary func([]float64) float64, lowest bool, duration time.Duration) api.SeriesList {
	slots := int(duration / list.Timerange.Resolution())
	if slots <= 0 {
		slots = 1
	}
	if slots > list.Timerange.Slots() {
		slots = list.Timerange.Slots()
	}
	sorted, values := sortSeries(list.Series, func(values []float64) float64 {
		return summary(values[len(values)-slots:])
	}, lowest)

	result := []api.Timeseries{}
	for i := range sorted {
		// Since the series are sorted, once one of them falls outside the threshold, we can stop.
		if (lowest && values[i] > threshold) || (!lowest && values[i] < threshold) {
			break
		}
		result = append(result, sorted[i])
	}

	return api.SeriesList{
		Series:    result,
		Timerange: list.Timerange,
	}
}
<|endoftext|>


<commit_before>/**
 * (C) Copyright 2013, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import (
	"fmt"
	"math"
	"testing"
	"net/http"
	"encoding/json"
)

type RecordingResponseWriter struct {
	header http.Header
	HeaderCode int
	Data []byte
}

func (self *RecordingResponseWriter) reset() {
	self.header = make(map[string][]string)
	self.HeaderCode = 0
	self.Data = nil
}

func (self *RecordingResponseWriter) Header() http.Header {
	return self.header
}

func (self *RecordingResponseWriter) Write(data []byte) (int, error) {
	self.Data = append(self.Data, data...)
	return len(data), nil
}

func NewRecordingResponseWriter() *RecordingResponseWriter {
	return &RecordingResponseWriter{ header : make(map[string][]string) }
}

func (self *RecordingResponseWriter) WriteHeader(code int) {
	self.HeaderCode = code
}

type testJsonStruct struct {
	String string
	Boolean bool
	Number float64
}

func TestJsonEncodeAndWriteResponse(t *testing.T) {

	response := NewRecordingResponseWriter()

	test := &testJsonStruct{ String: "test", Boolean: true, Number: math.MaxFloat64 }

	// Write the data
	if err := JsonEncodeAndWriteResponse(response, test); err != nil {
		t.Errorf("JsonEncodeAndWriteResponse is broken - %v", err)
	}

	// Ensure the response
	decoded := &testJsonStruct{}
	if err := json.Unmarshal(response.Data, decoded); err != nil {
		t.Errorf("JsonEncodeAndWriteResponse unmarshal data is broken - %v", err)
	}

	if test.String != decoded.String {
		t.Errorf("JsonEncodeAndWriteResponse is broken - expected string: %s - received: %s", test.String, decoded.String)
	}

	if test.Boolean != decoded.Boolean {
		t.Errorf("JsonEncodeAndWriteResponse is broken - expected bool : %s - received: %s", test.Boolean, decoded.Boolean)
	}

	if test.Number != decoded.Number {
		t.Errorf("JsonEncodeAndWriteResponse is broken - expected number : %s - received: %s", test.Number, decoded.Number)
	}

}

func TestWriteOkResponseString(t *testing.T) {

	response := NewRecordingResponseWriter()

	if err := WriteOkResponseString(response, "test"); err != nil {
		t.Errorf("IsHttpMethodPost is broken - %v", err)
	}

	if string(response.Data) != "test" {
		t.Errorf("IsHttpMethodPost is broken")
	}

	if response.Header().Get("Content-Type") != "text/plain; charset=utf-8" {
		t.Errorf("IsHttpMethodPost Content-Type is broken")
	}

	response.reset()

	if response.Header().Get("Content-Type") != "" {
		t.Errorf("IsHttpMethodPost reset is broken")
	}

	err := WriteOkResponseString(response, "")
	if err == nil {
		t.Errorf("WriteOkResponseString is broken - no error on empty message")
	}

	//writeOkResponseStringEmptyMsgPanic(response, t)

	if err := WriteOkResponseString(response, "t"); err != nil {
		t.Errorf("IsHttpMethodPost is broken - %v", err)
	}

	if string(response.Data) != "t" {
		t.Errorf("IsHttpMethodPost is broken")
	}

	response.reset()
	if err := WriteOkResponseString(response, "tttttttttttttttttttttttttttttttttttttttttttttttt"); err != nil {
		t.Errorf("IsHttpMethodPost is broken - %v", err)
	}

	// This will panic
	err = WriteOkResponseString(nil, "")
	if err == nil {
		t.Errorf("WriteOkResponseString is broken  - no error on nil response param")
	}
}

func TestIsHttpMethodPost(t *testing.T) {

	if IsHttpMethodPost(&http.Request{ Method : "" }) {
		t.Errorf("IsHttpMethodPost is broken")
	}

	if IsHttpMethodPost(&http.Request{ Method : "wrong" }) {
		t.Errorf("IsHttpMethodPost is broken")
	}

	if !IsHttpMethodPost(&http.Request{ Method : "post" }) {
		t.Errorf("IsHttpMethodPost is broken")
	}

	if !IsHttpMethodPost(&http.Request{ Method : "Post" }) {
		t.Errorf("IsHttpMethodPost is broken")
	}

	if !IsHttpMethodPost(&http.Request{ Method : "POST" }) {
		t.Errorf("IsHttpMethodPost is broken")
	}

	if !IsHttpMethodPost(&http.Request{ Method : "PosT" }) {
		t.Errorf("IsHttpMethodPost is broken")
	}

	// Verify the panic
	defer func() {
		if r := recover(); r == nil {
			t.Errorf("IsHttpMethodPost is broken - it did not panic on nil request")
		}
	}()

	// This method will panic.
	IsHttpMethodPost(nil)
}

<commit_msg>fixed compile<commit_after>/**
 * (C) Copyright 2013, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import (
	"math"
	"testing"
	"net/http"
	"encoding/json"
)

type RecordingResponseWriter struct {
	header http.Header
	HeaderCode int
	Data []byte
}

func (self *RecordingResponseWriter) reset() {
	self.header = make(map[string][]string)
	self.HeaderCode = 0
	self.Data = nil
}

func (self *RecordingResponseWriter) Header() http.Header {
	return self.header
}

func (self *RecordingResponseWriter) Write(data []byte) (int, error) {
	self.Data = append(self.Data, data...)
	return len(data), nil
}

func NewRecordingResponseWriter() *RecordingResponseWriter {
	return &RecordingResponseWriter{ header : make(map[string][]string) }
}

func (self *RecordingResponseWriter) WriteHeader(code int) {
	self.HeaderCode = code
}

type testJsonStruct struct {
	String string
	Boolean bool
	Number float64
}

func TestJsonEncodeAndWriteResponse(t *testing.T) {

	response := NewRecordingResponseWriter()

	test := &testJsonStruct{ String: "test", Boolean: true, Number: math.MaxFloat64 }

	// Write the data
	if err := JsonEncodeAndWriteResponse(response, test); err != nil {
		t.Errorf("JsonEncodeAndWriteResponse is broken - %v", err)
	}

	// Ensure the response
	decoded := &testJsonStruct{}
	if err := json.Unmarshal(response.Data, decoded); err != nil {
		t.Errorf("JsonEncodeAndWriteResponse unmarshal data is broken - %v", err)
	}

	if test.String != decoded.String {
		t.Errorf("JsonEncodeAndWriteResponse is broken - expected string: %s - received: %s", test.String, decoded.String)
	}

	if test.Boolean != decoded.Boolean {
		t.Errorf("JsonEncodeAndWriteResponse is broken - expected bool : %s - received: %s", test.Boolean, decoded.Boolean)
	}

	if test.Number != decoded.Number {
		t.Errorf("JsonEncodeAndWriteResponse is broken - expected number : %s - received: %s", test.Number, decoded.Number)
	}

}

func TestWriteOkResponseString(t *testing.T) {

	response := NewRecordingResponseWriter()

	if err := WriteOkResponseString(response, "test"); err != nil {
		t.Errorf("IsHttpMethodPost is broken - %v", err)
	}

	if string(response.Data) != "test" {
		t.Errorf("IsHttpMethodPost is broken")
	}

	if response.Header().Get("Content-Type") != "text/plain; charset=utf-8" {
		t.Errorf("IsHttpMethodPost Content-Type is broken")
	}

	response.reset()

	if response.Header().Get("Content-Type") != "" {
		t.Errorf("IsHttpMethodPost reset is broken")
	}

	err := WriteOkResponseString(response, "")
	if err == nil {
		t.Errorf("WriteOkResponseString is broken - no error on empty message")
	}

	//writeOkResponseStringEmptyMsgPanic(response, t)

	if err := WriteOkResponseString(response, "t"); err != nil {
		t.Errorf("IsHttpMethodPost is broken - %v", err)
	}

	if string(response.Data) != "t" {
		t.Errorf("IsHttpMethodPost is broken")
	}

	response.reset()
	if err := WriteOkResponseString(response, "tttttttttttttttttttttttttttttttttttttttttttttttt"); err != nil {
		t.Errorf("IsHttpMethodPost is broken - %v", err)
	}

	// This will panic
	err = WriteOkResponseString(nil, "")
	if err == nil {
		t.Errorf("WriteOkResponseString is broken  - no error on nil response param")
	}
}

func TestIsHttpMethodPost(t *testing.T) {

	if IsHttpMethodPost(&http.Request{ Method : "" }) {
		t.Errorf("IsHttpMethodPost is broken")
	}

	if IsHttpMethodPost(&http.Request{ Method : "wrong" }) {
		t.Errorf("IsHttpMethodPost is broken")
	}

	if !IsHttpMethodPost(&http.Request{ Method : "post" }) {
		t.Errorf("IsHttpMethodPost is broken")
	}

	if !IsHttpMethodPost(&http.Request{ Method : "Post" }) {
		t.Errorf("IsHttpMethodPost is broken")
	}

	if !IsHttpMethodPost(&http.Request{ Method : "POST" }) {
		t.Errorf("IsHttpMethodPost is broken")
	}

	if !IsHttpMethodPost(&http.Request{ Method : "PosT" }) {
		t.Errorf("IsHttpMethodPost is broken")
	}

	// Verify the panic
	defer func() {
		if r := recover(); r == nil {
			t.Errorf("IsHttpMethodPost is broken - it did not panic on nil request")
		}
	}()

	// This method will panic.
	IsHttpMethodPost(nil)
}

<|endoftext|>


<commit_before>package byline

import (
	"bufio"
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"regexp"
)

var (
	// ErrOmitLine - error for Map*Err/AWKMode, for omitting current line
	ErrOmitLine = errors.New("ErrOmitLine")

	// default field separator
	defaultFS = regexp.MustCompile(`\s+`)
	// default line separator
	defaultRS byte = '\n'
	// for Grep* methods
	nullBytes = []byte{}
	// bytes.Buffer growth to this limit
	bufferSizeLimit = 1024
)

// Reader - line by line Reader
type Reader struct {
	scanner     *bufio.Scanner
	buffer      *bytes.Buffer
	existsData  bool
	filterFuncs []func(line []byte) ([]byte, error)
	awkVars     AWKVars
}

// AWKVars - settings for AWK mode, see man awk
type AWKVars struct {
	NR int            // number of current line (begin from 1)
	NF int            // fields count in curent line
	RS byte           // record separator, default is '\n'
	FS *regexp.Regexp // field separator, default is `\s+`
}

// NewReader - get new line by line Reader
func NewReader(reader io.Reader) *Reader {
	lr := &Reader{
		scanner:    bufio.NewScanner(reader),
		buffer:     bytes.NewBuffer([]byte{}),
		existsData: true,
		awkVars: AWKVars{
			RS: defaultRS,
			FS: defaultFS,
		},
	}

	lr.scanner.Split(lr.scanLinesWithNL)
	lr.buffer.Grow(bufferSizeLimit)

	return lr
}

func (lr *Reader) scanLinesWithNL(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}
	if i := bytes.IndexByte(data, lr.awkVars.RS); i >= 0 {
		// We have a full newline-terminated line.
		return i + 1, data[0 : i+1], nil
	}
	// If we're at EOF, we have a final, non-terminated line. Return it.
	if atEOF {
		return len(data), data, nil
	}

	// Request more data.
	return 0, nil, nil
}

// Read - implement io.Reader interface
func (lr *Reader) Read(p []byte) (n int, err error) {
	var (
		bufErr, filterErr error
		lineBytes         []byte
	)

	for lr.existsData && bufErr == nil && lr.buffer.Len() < bufferSizeLimit {
		if lr.existsData = lr.scanner.Scan(); !lr.existsData {
			break
		}

		lineBytes = lr.scanner.Bytes()
		lr.awkVars.NR++

		for _, filterFunc := range lr.filterFuncs {
			lineBytes, filterErr = filterFunc(lineBytes)
			if filterErr != nil {
				switch filterErr {
				case ErrOmitLine:
					lineBytes = nullBytes
				default:
					bufErr = filterErr
				}
				break
			}
		}

		_, _ = lr.buffer.Write(lineBytes) // #nosec - err always is nil
	}

	if !lr.existsData && bufErr == nil {
		bufErr = lr.scanner.Err()
	}

	n, err = lr.buffer.Read(p)
	if err != nil && bufErr == nil {
		bufErr = err
	}

	return n, bufErr
}

// Map - set filter function for process each line
func (lr *Reader) Map(filterFn func([]byte) []byte) *Reader {
	return lr.MapErr(func(line []byte) ([]byte, error) {
		return filterFn(line), nil
	})
}

// MapErr - set filter function for process each line, returns error if needed (io.EOF for example)
func (lr *Reader) MapErr(filterFn func([]byte) ([]byte, error)) *Reader {
	lr.filterFuncs = append(lr.filterFuncs, filterFn)
	return lr
}

// MapString - set filter function for process each line as string
func (lr *Reader) MapString(filterFn func(string) string) *Reader {
	return lr.MapErr(func(line []byte) ([]byte, error) {
		return []byte(filterFn(string(line))), nil
	})
}

// MapStringErr - set filter function for process each line as string, returns error if needed (io.EOF for example)
func (lr *Reader) MapStringErr(filterFn func(string) (string, error)) *Reader {
	return lr.MapErr(func(line []byte) ([]byte, error) {
		newString, err := filterFn(string(line))
		return []byte(newString), err
	})
}

// Grep - grep lines by func
func (lr *Reader) Grep(filterFn func([]byte) bool) *Reader {
	return lr.MapErr(func(line []byte) ([]byte, error) {
		if filterFn(line) {
			return line, nil
		}

		return nullBytes, ErrOmitLine
	})
}

// GrepString - grep lines as string by func
func (lr *Reader) GrepString(filterFn func(string) bool) *Reader {
	return lr.Grep(func(line []byte) bool {
		return filterFn(string(line))
	})
}

// GrepByRegexp - grep lines by regexp
func (lr *Reader) GrepByRegexp(re *regexp.Regexp) *Reader {
	return lr.Grep(func(line []byte) bool {
		return re.Match(line)
	})
}

// SetRS - set lines (records) separator
func (lr *Reader) SetRS(rs byte) *Reader {
	lr.awkVars.RS = rs
	return lr
}

// SetFS - set field separator for AWK mode
func (lr *Reader) SetFS(fs *regexp.Regexp) *Reader {
	lr.awkVars.FS = fs
	return lr
}

// AWKMode - process lines with AWK like mode
func (lr *Reader) AWKMode(filterFn func(line string, fields []string, vars AWKVars) (string, error)) *Reader {
	return lr.MapErr(func(line []byte) ([]byte, error) {
		addRS := false
		RS := []byte{lr.awkVars.RS}
		if bytes.HasSuffix(line, RS) {
			addRS = true
			line = bytes.TrimSuffix(line, RS)
		}

		lineStr := string(line)
		fields := lr.awkVars.FS.Split(lineStr, -1)
		lr.awkVars.NF = len(fields)
		result, err := filterFn(lineStr, fields, lr.awkVars)
		if err != nil {
			return nullBytes, err
		}

		resultBytes := []byte(result)
		if !bytes.HasSuffix(resultBytes, RS) && addRS {
			resultBytes = append(resultBytes, lr.awkVars.RS)
		}
		return resultBytes, nil
	})
}

// Discard - read all content from Reader for side effect from filter functions
func (lr *Reader) Discard() error {
	_, err := io.Copy(ioutil.Discard, lr)
	return err
}

// ReadAllSlice - read all content from Reader by lines to slice of []byte
func (lr *Reader) ReadAllSlice() ([][]byte, error) {
	result := [][]byte{}
	err := lr.Map(func(line []byte) []byte {
		result = append(result, line)
		return nullBytes
	}).Discard()

	return result, err
}

// ReadAll - read all content from Reader to slice of bytes
func (lr *Reader) ReadAll() ([]byte, error) {
	return ioutil.ReadAll(lr)
}

// ReadAllSliceString - read all content from Reader to string slice by lines
func (lr *Reader) ReadAllSliceString() ([]string, error) {
	result := []string{}
	err := lr.MapString(func(line string) string {
		result = append(result, line)
		return ""
	}).Discard()

	return result, err
}

// ReadAllString - read all content from Reader to one string
func (lr *Reader) ReadAllString() (string, error) {
	result, err := ioutil.ReadAll(lr)
	return string(result), err
}
<commit_msg>Renamed private method<commit_after>package byline

import (
	"bufio"
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"regexp"
)

var (
	// ErrOmitLine - error for Map*Err/AWKMode, for omitting current line
	ErrOmitLine = errors.New("ErrOmitLine")

	// default field separator
	defaultFS = regexp.MustCompile(`\s+`)
	// default line separator
	defaultRS byte = '\n'
	// for Grep* methods
	nullBytes = []byte{}
	// bytes.Buffer growth to this limit
	bufferSizeLimit = 1024
)

// Reader - line by line Reader
type Reader struct {
	scanner     *bufio.Scanner
	buffer      *bytes.Buffer
	existsData  bool
	filterFuncs []func(line []byte) ([]byte, error)
	awkVars     AWKVars
}

// AWKVars - settings for AWK mode, see man awk
type AWKVars struct {
	NR int            // number of current line (begin from 1)
	NF int            // fields count in curent line
	RS byte           // record separator, default is '\n'
	FS *regexp.Regexp // field separator, default is `\s+`
}

// NewReader - get new line by line Reader
func NewReader(reader io.Reader) *Reader {
	lr := &Reader{
		scanner:    bufio.NewScanner(reader),
		buffer:     bytes.NewBuffer([]byte{}),
		existsData: true,
		awkVars: AWKVars{
			RS: defaultRS,
			FS: defaultFS,
		},
	}

	lr.scanner.Split(lr.scanLinesBySep)
	lr.buffer.Grow(bufferSizeLimit)

	return lr
}

func (lr *Reader) scanLinesBySep(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}
	if i := bytes.IndexByte(data, lr.awkVars.RS); i >= 0 {
		// We have a full RS-terminated line.
		return i + 1, data[0 : i+1], nil
	}
	// If we're at EOF, we have a final, non-terminated line. Return it.
	if atEOF {
		return len(data), data, nil
	}

	// Request more data.
	return 0, nil, nil
}

// Read - implement io.Reader interface
func (lr *Reader) Read(p []byte) (n int, err error) {
	var (
		bufErr, filterErr error
		lineBytes         []byte
	)

	for lr.existsData && bufErr == nil && lr.buffer.Len() < bufferSizeLimit {
		if lr.existsData = lr.scanner.Scan(); !lr.existsData {
			break
		}

		lineBytes = lr.scanner.Bytes()
		lr.awkVars.NR++

		for _, filterFunc := range lr.filterFuncs {
			lineBytes, filterErr = filterFunc(lineBytes)
			if filterErr != nil {
				switch filterErr {
				case ErrOmitLine:
					lineBytes = nullBytes
				default:
					bufErr = filterErr
				}
				break
			}
		}

		_, _ = lr.buffer.Write(lineBytes) // #nosec - err always is nil
	}

	if !lr.existsData && bufErr == nil {
		bufErr = lr.scanner.Err()
	}

	n, err = lr.buffer.Read(p)
	if err != nil && bufErr == nil {
		bufErr = err
	}

	return n, bufErr
}

// Map - set filter function for process each line
func (lr *Reader) Map(filterFn func([]byte) []byte) *Reader {
	return lr.MapErr(func(line []byte) ([]byte, error) {
		return filterFn(line), nil
	})
}

// MapErr - set filter function for process each line, returns error if needed (io.EOF for example)
func (lr *Reader) MapErr(filterFn func([]byte) ([]byte, error)) *Reader {
	lr.filterFuncs = append(lr.filterFuncs, filterFn)
	return lr
}

// MapString - set filter function for process each line as string
func (lr *Reader) MapString(filterFn func(string) string) *Reader {
	return lr.MapErr(func(line []byte) ([]byte, error) {
		return []byte(filterFn(string(line))), nil
	})
}

// MapStringErr - set filter function for process each line as string, returns error if needed (io.EOF for example)
func (lr *Reader) MapStringErr(filterFn func(string) (string, error)) *Reader {
	return lr.MapErr(func(line []byte) ([]byte, error) {
		newString, err := filterFn(string(line))
		return []byte(newString), err
	})
}

// Grep - grep lines by func
func (lr *Reader) Grep(filterFn func([]byte) bool) *Reader {
	return lr.MapErr(func(line []byte) ([]byte, error) {
		if filterFn(line) {
			return line, nil
		}

		return nullBytes, ErrOmitLine
	})
}

// GrepString - grep lines as string by func
func (lr *Reader) GrepString(filterFn func(string) bool) *Reader {
	return lr.Grep(func(line []byte) bool {
		return filterFn(string(line))
	})
}

// GrepByRegexp - grep lines by regexp
func (lr *Reader) GrepByRegexp(re *regexp.Regexp) *Reader {
	return lr.Grep(func(line []byte) bool {
		return re.Match(line)
	})
}

// SetRS - set lines (records) separator
func (lr *Reader) SetRS(rs byte) *Reader {
	lr.awkVars.RS = rs
	return lr
}

// SetFS - set field separator for AWK mode
func (lr *Reader) SetFS(fs *regexp.Regexp) *Reader {
	lr.awkVars.FS = fs
	return lr
}

// AWKMode - process lines with AWK like mode
func (lr *Reader) AWKMode(filterFn func(line string, fields []string, vars AWKVars) (string, error)) *Reader {
	return lr.MapErr(func(line []byte) ([]byte, error) {
		addRS := false
		RS := []byte{lr.awkVars.RS}
		if bytes.HasSuffix(line, RS) {
			addRS = true
			line = bytes.TrimSuffix(line, RS)
		}

		lineStr := string(line)
		fields := lr.awkVars.FS.Split(lineStr, -1)
		lr.awkVars.NF = len(fields)
		result, err := filterFn(lineStr, fields, lr.awkVars)
		if err != nil {
			return nullBytes, err
		}

		resultBytes := []byte(result)
		if !bytes.HasSuffix(resultBytes, RS) && addRS {
			resultBytes = append(resultBytes, lr.awkVars.RS)
		}
		return resultBytes, nil
	})
}

// Discard - read all content from Reader for side effect from filter functions
func (lr *Reader) Discard() error {
	_, err := io.Copy(ioutil.Discard, lr)
	return err
}

// ReadAllSlice - read all content from Reader by lines to slice of []byte
func (lr *Reader) ReadAllSlice() ([][]byte, error) {
	result := [][]byte{}
	err := lr.Map(func(line []byte) []byte {
		result = append(result, line)
		return nullBytes
	}).Discard()

	return result, err
}

// ReadAll - read all content from Reader to slice of bytes
func (lr *Reader) ReadAll() ([]byte, error) {
	return ioutil.ReadAll(lr)
}

// ReadAllSliceString - read all content from Reader to string slice by lines
func (lr *Reader) ReadAllSliceString() ([]string, error) {
	result := []string{}
	err := lr.MapString(func(line string) string {
		result = append(result, line)
		return ""
	}).Discard()

	return result, err
}

// ReadAllString - read all content from Reader to one string
func (lr *Reader) ReadAllString() (string, error) {
	result, err := ioutil.ReadAll(lr)
	return string(result), err
}
<|endoftext|>