<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package math

// The original C code, the long comment, and the constants
// below are from FreeBSD's /usr/src/lib/msun/src/s_log1p.c
// and came with this notice. The go code is a simplified
// version of the original C.
//
// ====================================================
// Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved.
//
// Developed at SunPro, a Sun Microsystems, Inc. business.
// Permission to use, copy, modify, and distribute this
// software is freely granted, provided that this notice
// is preserved.
// ====================================================
//
//
// double log1p(double x)
//
// Method :
//   1. Argument Reduction: find k and f such that
//                      1+x = 2**k * (1+f),
//         where  sqrt(2)/2 < 1+f < sqrt(2) .
//
//      Note. If k=0, then f=x is exact. However, if k!=0, then f
//      may not be representable exactly. In that case, a correction
//      term is need. Let u=1+x rounded. Let c = (1+x)-u, then
//      log(1+x) - log(u) ~ c/u. Thus, we proceed to compute log(u),
//      and add back the correction term c/u.
//      (Note: when x > 2**53, one can simply return log(x))
//
//   2. Approximation of log1p(f).
//      Let s = f/(2+f) ; based on log(1+f) = log(1+s) - log(1-s)
//               = 2s + 2/3 s**3 + 2/5 s**5 + .....,
//               = 2s + s*R
//      We use a special Reme algorithm on [0,0.1716] to generate
//      a polynomial of degree 14 to approximate R The maximum error
//      of this polynomial approximation is bounded by 2**-58.45. In
//      other words,
//                      2      4      6      8      10      12      14
//          R(z) ~ Lp1*s +Lp2*s +Lp3*s +Lp4*s +Lp5*s  +Lp6*s  +Lp7*s
//      (the values of Lp1 to Lp7 are listed in the program)
//      and
//          |      2          14          |     -58.45
//          | Lp1*s +...+Lp7*s    -  R(z) | <= 2
//          |                             |
//      Note that 2s = f - s*f = f - hfsq + s*hfsq, where hfsq = f*f/2.
//      In order to guarantee error in log below 1ulp, we compute log
//      by
//              log1p(f) = f - (hfsq - s*(hfsq+R)).
//
//   3. Finally, log1p(x) = k*ln2 + log1p(f).
//                        = k*ln2_hi+(f-(hfsq-(s*(hfsq+R)+k*ln2_lo)))
//      Here ln2 is split into two floating point number:
//                   ln2_hi + ln2_lo,
//      where n*ln2_hi is always exact for |n| < 2000.
//
// Special cases:
//      log1p(x) is NaN with signal if x < -1 (including -INF) ;
//      log1p(+INF) is +INF; log1p(-1) is -INF with signal;
//      log1p(NaN) is that NaN with no signal.
//
// Accuracy:
//      according to an error analysis, the error is always less than
//      1 ulp (unit in the last place).
//
// Constants:
// The hexadecimal values are the intended ones for the following
// constants. The decimal values may be used, provided that the
// compiler will convert from decimal to binary accurately enough
// to produce the hexadecimal values shown.
//
// Note: Assuming log() return accurate answer, the following
//       algorithm can be used to compute log1p(x) to within a few ULP:
//
//              u = 1+x;
//              if(u==1.0) return x ; else
//                         return log(u)*(x/(u-1.0));
//
//       See HP-15C Advanced Functions Handbook, p.193.

// Log1p returns the natural logarithm of 1 plus its argument x.
// It is more accurate than Log(1 + x) when x is near zero.
//
// Special cases are:
//	Log1p(+Inf) = +Inf
//	Log1p(±0) = ±0
//	Log1p(-1) = -Inf
//	Log1p(x < -1) = NaN
//	Log1p(NaN) = NaN
func Log1p(x float64) float64

func log1p(x float64) float64 {
	const (
		Sqrt2M1     = 4.142135623730950488017e-01  // Sqrt(2)-1 = 0x3fda827999fcef34
		Sqrt2HalfM1 = -2.928932188134524755992e-01 // Sqrt(2)/2-1 = 0xbfd2bec333018866
		Small       = 1.0 / (1 << 29)              // 2**-29 = 0x3e20000000000000
		Tiny        = 1.0 / (1 << 54)              // 2**-54
		Two53       = 1 << 53                      // 2**53
		Ln2Hi       = 6.93147180369123816490e-01   // 3fe62e42fee00000
		Ln2Lo       = 1.90821492927058770002e-10   // 3dea39ef35793c76
		Lp1         = 6.666666666666735130e-01     // 3FE5555555555593
		Lp2         = 3.999999999940941908e-01     // 3FD999999997FA04
		Lp3         = 2.857142874366239149e-01     // 3FD2492494229359
		Lp4         = 2.222219843214978396e-01     // 3FCC71C51D8E78AF
		Lp5         = 1.818357216161805012e-01     // 3FC7466496CB03DE
		Lp6         = 1.531383769920937332e-01     // 3FC39A09D078C69F
		Lp7         = 1.479819860511658591e-01     // 3FC2F112DF3E5244
	)

	// special cases
	switch {
	case x < -1 || IsNaN(x): // includes -Inf
		return NaN()
	case x == -1:
		return Inf(-1)
	case IsInf(x, 1):
		return Inf(1)
	}

	absx := x
	if absx < 0 {
		absx = -absx
	}

	var f float64
	var iu uint64
	k := 1
	if absx < Sqrt2M1 { //  |x| < Sqrt(2)-1
		if absx < Small { // |x| < 2**-29
			if absx < Tiny { // |x| < 2**-54
				return x
			}
			return x - x*x*0.5
		}
		if x > Sqrt2HalfM1 { // Sqrt(2)/2-1 < x
			// (Sqrt(2)/2-1) < x < (Sqrt(2)-1)
			k = 0
			f = x
			iu = 1
		}
	}
	var c float64
	if k != 0 {
		var u float64
		if absx < Two53 { // 1<<53
			u = 1.0 + x
			iu = Float64bits(u)
			k = int((iu >> 52) - 1023)
			// correction term
			if k > 0 {
				c = 1.0 - (u - x)
			} else {
				c = x - (u - 1.0)
			}
			c /= u
		} else {
			u = x
			iu = Float64bits(u)
			k = int((iu >> 52) - 1023)
			c = 0
		}
		iu &= 0x000fffffffffffff
		if iu < 0x0006a09e667f3bcd { // mantissa of Sqrt(2)
			u = Float64frombits(iu | 0x3ff0000000000000) // normalize u
		} else {
			k++
			u = Float64frombits(iu | 0x3fe0000000000000) // normalize u/2
			iu = (0x0010000000000000 - iu) >> 2
		}
		f = u - 1.0 // Sqrt(2)/2 < u < Sqrt(2)
	}
	hfsq := 0.5 * f * f
	var s, R, z float64
	if iu == 0 { // |f| < 2**-20
		if f == 0 {
			if k == 0 {
				return 0
			}
			c += float64(k) * Ln2Lo
			return float64(k)*Ln2Hi + c
		}
		R = hfsq * (1.0 - 0.66666666666666666*f) // avoid division
		if k == 0 {
			return f - R
		}
		return float64(k)*Ln2Hi - ((R - (float64(k)*Ln2Lo + c)) - f)
	}
	s = f / (2.0 + f)
	z = s * s
	R = z * (Lp1 + z*(Lp2+z*(Lp3+z*(Lp4+z*(Lp5+z*(Lp6+z*Lp7))))))
	if k == 0 {
		return f - (hfsq - s*(hfsq+R))
	}
	return float64(k)*Ln2Hi - ((hfsq - (s*(hfsq+R) + (float64(k)*Ln2Lo + c))) - f)
}
<commit_msg>math: simplify the code<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package math

// The original C code, the long comment, and the constants
// below are from FreeBSD's /usr/src/lib/msun/src/s_log1p.c
// and came with this notice. The go code is a simplified
// version of the original C.
//
// ====================================================
// Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved.
//
// Developed at SunPro, a Sun Microsystems, Inc. business.
// Permission to use, copy, modify, and distribute this
// software is freely granted, provided that this notice
// is preserved.
// ====================================================
//
//
// double log1p(double x)
//
// Method :
//   1. Argument Reduction: find k and f such that
//                      1+x = 2**k * (1+f),
//         where  sqrt(2)/2 < 1+f < sqrt(2) .
//
//      Note. If k=0, then f=x is exact. However, if k!=0, then f
//      may not be representable exactly. In that case, a correction
//      term is need. Let u=1+x rounded. Let c = (1+x)-u, then
//      log(1+x) - log(u) ~ c/u. Thus, we proceed to compute log(u),
//      and add back the correction term c/u.
//      (Note: when x > 2**53, one can simply return log(x))
//
//   2. Approximation of log1p(f).
//      Let s = f/(2+f) ; based on log(1+f) = log(1+s) - log(1-s)
//               = 2s + 2/3 s**3 + 2/5 s**5 + .....,
//               = 2s + s*R
//      We use a special Reme algorithm on [0,0.1716] to generate
//      a polynomial of degree 14 to approximate R The maximum error
//      of this polynomial approximation is bounded by 2**-58.45. In
//      other words,
//                      2      4      6      8      10      12      14
//          R(z) ~ Lp1*s +Lp2*s +Lp3*s +Lp4*s +Lp5*s  +Lp6*s  +Lp7*s
//      (the values of Lp1 to Lp7 are listed in the program)
//      and
//          |      2          14          |     -58.45
//          | Lp1*s +...+Lp7*s    -  R(z) | <= 2
//          |                             |
//      Note that 2s = f - s*f = f - hfsq + s*hfsq, where hfsq = f*f/2.
//      In order to guarantee error in log below 1ulp, we compute log
//      by
//              log1p(f) = f - (hfsq - s*(hfsq+R)).
//
//   3. Finally, log1p(x) = k*ln2 + log1p(f).
//                        = k*ln2_hi+(f-(hfsq-(s*(hfsq+R)+k*ln2_lo)))
//      Here ln2 is split into two floating point number:
//                   ln2_hi + ln2_lo,
//      where n*ln2_hi is always exact for |n| < 2000.
//
// Special cases:
//      log1p(x) is NaN with signal if x < -1 (including -INF) ;
//      log1p(+INF) is +INF; log1p(-1) is -INF with signal;
//      log1p(NaN) is that NaN with no signal.
//
// Accuracy:
//      according to an error analysis, the error is always less than
//      1 ulp (unit in the last place).
//
// Constants:
// The hexadecimal values are the intended ones for the following
// constants. The decimal values may be used, provided that the
// compiler will convert from decimal to binary accurately enough
// to produce the hexadecimal values shown.
//
// Note: Assuming log() return accurate answer, the following
//       algorithm can be used to compute log1p(x) to within a few ULP:
//
//              u = 1+x;
//              if(u==1.0) return x ; else
//                         return log(u)*(x/(u-1.0));
//
//       See HP-15C Advanced Functions Handbook, p.193.

// Log1p returns the natural logarithm of 1 plus its argument x.
// It is more accurate than Log(1 + x) when x is near zero.
//
// Special cases are:
//	Log1p(+Inf) = +Inf
//	Log1p(±0) = ±0
//	Log1p(-1) = -Inf
//	Log1p(x < -1) = NaN
//	Log1p(NaN) = NaN
func Log1p(x float64) float64

func log1p(x float64) float64 {
	const (
		Sqrt2M1     = 4.142135623730950488017e-01  // Sqrt(2)-1 = 0x3fda827999fcef34
		Sqrt2HalfM1 = -2.928932188134524755992e-01 // Sqrt(2)/2-1 = 0xbfd2bec333018866
		Small       = 1.0 / (1 << 29)              // 2**-29 = 0x3e20000000000000
		Tiny        = 1.0 / (1 << 54)              // 2**-54
		Two53       = 1 << 53                      // 2**53
		Ln2Hi       = 6.93147180369123816490e-01   // 3fe62e42fee00000
		Ln2Lo       = 1.90821492927058770002e-10   // 3dea39ef35793c76
		Lp1         = 6.666666666666735130e-01     // 3FE5555555555593
		Lp2         = 3.999999999940941908e-01     // 3FD999999997FA04
		Lp3         = 2.857142874366239149e-01     // 3FD2492494229359
		Lp4         = 2.222219843214978396e-01     // 3FCC71C51D8E78AF
		Lp5         = 1.818357216161805012e-01     // 3FC7466496CB03DE
		Lp6         = 1.531383769920937332e-01     // 3FC39A09D078C69F
		Lp7         = 1.479819860511658591e-01     // 3FC2F112DF3E5244
	)

	// special cases
	switch {
	case x < -1 || IsNaN(x): // includes -Inf
		return NaN()
	case x == -1:
		return Inf(-1)
	case IsInf(x, 1):
		return Inf(1)
	}

	absx := Abs(x)

	var f float64
	var iu uint64
	k := 1
	if absx < Sqrt2M1 { //  |x| < Sqrt(2)-1
		if absx < Small { // |x| < 2**-29
			if absx < Tiny { // |x| < 2**-54
				return x
			}
			return x - x*x*0.5
		}
		if x > Sqrt2HalfM1 { // Sqrt(2)/2-1 < x
			// (Sqrt(2)/2-1) < x < (Sqrt(2)-1)
			k = 0
			f = x
			iu = 1
		}
	}
	var c float64
	if k != 0 {
		var u float64
		if absx < Two53 { // 1<<53
			u = 1.0 + x
			iu = Float64bits(u)
			k = int((iu >> 52) - 1023)
			// correction term
			if k > 0 {
				c = 1.0 - (u - x)
			} else {
				c = x - (u - 1.0)
			}
			c /= u
		} else {
			u = x
			iu = Float64bits(u)
			k = int((iu >> 52) - 1023)
			c = 0
		}
		iu &= 0x000fffffffffffff
		if iu < 0x0006a09e667f3bcd { // mantissa of Sqrt(2)
			u = Float64frombits(iu | 0x3ff0000000000000) // normalize u
		} else {
			k++
			u = Float64frombits(iu | 0x3fe0000000000000) // normalize u/2
			iu = (0x0010000000000000 - iu) >> 2
		}
		f = u - 1.0 // Sqrt(2)/2 < u < Sqrt(2)
	}
	hfsq := 0.5 * f * f
	var s, R, z float64
	if iu == 0 { // |f| < 2**-20
		if f == 0 {
			if k == 0 {
				return 0
			}
			c += float64(k) * Ln2Lo
			return float64(k)*Ln2Hi + c
		}
		R = hfsq * (1.0 - 0.66666666666666666*f) // avoid division
		if k == 0 {
			return f - R
		}
		return float64(k)*Ln2Hi - ((R - (float64(k)*Ln2Lo + c)) - f)
	}
	s = f / (2.0 + f)
	z = s * s
	R = z * (Lp1 + z*(Lp2+z*(Lp3+z*(Lp4+z*(Lp5+z*(Lp6+z*Lp7))))))
	if k == 0 {
		return f - (hfsq - s*(hfsq+R))
	}
	return float64(k)*Ln2Hi - ((hfsq - (s*(hfsq+R) + (float64(k)*Ln2Lo + c))) - f)
}
<|endoftext|>


<commit_before>// Copyright 2013 The go-instagram AUTHORS. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package instagram

import (
	"errors"
	"fmt"
	"net/url"
)

// TagsService handles communication with the tag related
// methods of the Instagram API.
//
// Instagram API docs: http://instagram.com/developer/endpoints/tags/
type TagsService struct {
	client *Client
}

// Tag represents information about a tag object.
type Tag struct {
	MediaCount int    `json:"media_count,omitempty"`
	Name       string `json:"name,omitempty"`
}

// Get information aout a tag object.
//
// Instagram API docs: http://instagram.com/developer/endpoints/tags/#get_tags
func (s *TagsService) Get(tagName string) (*Tag, error) {
	u := fmt.Sprintf("tags/%v", tagName)
	req, err := s.client.NewRequest("GET", u, "")
	if err != nil {
		return nil, err
	}

	tag := new(Tag)
	_, err = s.client.Do(req, tag)
	return tag, err
}

// RecentMedia Get a list of recently tagged media.
//
// Instagram API docs: http://instagram.com/developer/endpoints/tags/#get_tags_media_recent
func (s *TagsService) RecentMedia(tagName string, opt *Parameters) ([]Media, *ResponsePagination, error) {
	u := fmt.Sprintf("tags/%v/media/recent", tagName)
	if opt != nil {
		params := url.Values{}
		if opt.MinID != "" {
			params.Add("min_id", opt.MinID)
		}
		if opt.MaxID != "" {
			params.Add("max_id", opt.MaxID)
		}
		u += "?" + params.Encode()
	}
	req, err := s.client.NewRequest("GET", u, "")
	if err != nil {
		return nil, nil, err
	}

	media := new([]Media)

	_, err = s.client.Do(req, media)
	if err != nil {
		return nil, nil, errors.New(fmt.Sprintf("go-instagram Tag.RecentMedia error:%v on URL %s", err, req.URL.String()))
	}

	page := new(ResponsePagination)
	if s.client.Response.Pagination != nil {
		page = s.client.Response.Pagination
	}

	return *media, page, err
}

// Search for tags by name.
//
// Instagram API docs: http://instagram.com/developer/endpoints/tags/#get_tags_search
func (s *TagsService) Search(q string) ([]Tag, *ResponsePagination, error) {
	u := "tags/search?q=" + q
	req, err := s.client.NewRequest("GET", u, "")
	if err != nil {
		return nil, nil, err
	}

	tags := new([]Tag)

	_, err = s.client.Do(req, tags)
	if err != nil {
		return nil, nil, err
	}

	page := new(ResponsePagination)
	if s.client.Response.Pagination != nil {
		page = s.client.Response.Pagination
	}

	return *tags, page, err
}
<commit_msg>Early, catch attempts to search for disallowed hashtags, and block those attempts.<commit_after>// Copyright 2013 The go-instagram AUTHORS. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package instagram

import (
	"errors"
	"fmt"
	"net/url"
	"regexp"
)

// TagsService handles communication with the tag related
// methods of the Instagram API.
//
// Instagram API docs: http://instagram.com/developer/endpoints/tags/
type TagsService struct {
	client *Client
}

// Tag represents information about a tag object.
type Tag struct {
	MediaCount int    `json:"media_count,omitempty"`
	Name       string `json:"name,omitempty"`
}

// Get information aout a tag object.
//
// Instagram API docs: http://instagram.com/developer/endpoints/tags/#get_tags
func (s *TagsService) Get(tagName string) (*Tag, error) {
	u := fmt.Sprintf("tags/%v", tagName)
	req, err := s.client.NewRequest("GET", u, "")
	if err != nil {
		return nil, err
	}

	tag := new(Tag)
	_, err = s.client.Do(req, tag)
	return tag, err
}

// RecentMedia Get a list of recently tagged media.
//
// Instagram API docs: http://instagram.com/developer/endpoints/tags/#get_tags_media_recent
func (s *TagsService) RecentMedia(tagName string, opt *Parameters) ([]Media, *ResponsePagination, error) {
	valid, err := validTagName(tagName)
	if err != nil {
		return nil, nil, err
	}

	if !valid {
		//For now, I don't think this is an error but instead an early catch of an input that won't give a result
		//but it's not clearly defined (as far as I can tell) in the Instagram spec that it *couldn't* give a result
		//In future, this might change to give an error, though
		//return nil, nil, errors.New(`go-instagram Tag.RecentMedia error: Tag names must contain only alphabetical and numerical characters.`)
		return []Media{}, &ResponsePagination{}, nil
	}

	u := fmt.Sprintf("tags/%v/media/recent", tagName)
	if opt != nil {
		params := url.Values{}
		if opt.MinID != "" {
			params.Add("min_id", opt.MinID)
		}
		if opt.MaxID != "" {
			params.Add("max_id", opt.MaxID)
		}
		u += "?" + params.Encode()
	}
	req, err := s.client.NewRequest("GET", u, "")
	if err != nil {
		return nil, nil, err
	}

	media := new([]Media)

	_, err = s.client.Do(req, media)
	if err != nil {
		return nil, nil, errors.New(fmt.Sprintf("go-instagram Tag.RecentMedia error:%s on URL %s", err.Error(), req.URL.String()))
	}

	page := new(ResponsePagination)
	if s.client.Response.Pagination != nil {
		page = s.client.Response.Pagination
	}

	return *media, page, err
}

// Search for tags by name.
//
// Instagram API docs: http://instagram.com/developer/endpoints/tags/#get_tags_search
func (s *TagsService) Search(q string) ([]Tag, *ResponsePagination, error) {
	u := "tags/search?q=" + q
	req, err := s.client.NewRequest("GET", u, "")
	if err != nil {
		return nil, nil, err
	}

	tags := new([]Tag)

	_, err = s.client.Do(req, tags)
	if err != nil {
		return nil, nil, err
	}

	page := new(ResponsePagination)
	if s.client.Response.Pagination != nil {
		page = s.client.Response.Pagination
	}

	return *tags, page, err
}

// Strip out things we know Instagram won't accept. For example, hyphens.
func validTagName(tagName string) (bool, error) {
	//\W matches any non-word character
	reg, err := regexp.Compile(`\W`)
	if err != nil {
		return false, err
	}

	if reg.MatchString(tagName) {
		return false, nil
	}

	return true, nil
}
<|endoftext|>


<commit_before>package PEncoding

import (
	. "ParquetType"
	"bytes"
	"encoding/binary"
	"log"
	"parquet"
)

func ReadPlain(bytesReader *bytes.Reader, dataType parquet.Type, cnt uint64, bitWidth uint64) []interface{} {
	if dataType == parquet.Type_BOOLEAN {
		res := ReadBitPacked(bytesReader, uint64(cnt<<1), 1)
		return res
	} else if dataType == parquet.Type_INT32 {
		resTmp := ReadPlainINT32(bytesReader, cnt)
		res := make([]interface{}, len(resTmp))
		for i := 0; i < len(resTmp); i++ {
			res[i] = resTmp[i]
		}
		return res
	} else if dataType == parquet.Type_INT64 {
		resTmp := ReadPlainINT64(bytesReader, cnt)
		res := make([]interface{}, len(resTmp))
		for i := 0; i < len(resTmp); i++ {
			res[i] = resTmp[i]
		}
		return res
	} else if dataType == parquet.Type_INT96 {
		resTmp := ReadPlainINT96(bytesReader, cnt)
		res := make([]interface{}, len(resTmp))
		for i := 0; i < len(resTmp); i++ {
			res[i] = resTmp[i]
		}
		return res

	} else if dataType == parquet.Type_FLOAT {
		resTmp := ReadPlainFLOAT(bytesReader, cnt)
		res := make([]interface{}, len(resTmp))
		for i := 0; i < len(resTmp); i++ {
			res[i] = resTmp[i]
		}
		return res
	} else if dataType == parquet.Type_DOUBLE {
		resTmp := ReadPlainDOUBLE(bytesReader, cnt)
		res := make([]interface{}, len(resTmp))
		for i := 0; i < len(resTmp); i++ {
			res[i] = resTmp[i]
		}
		return res
	} else if dataType == parquet.Type_BYTE_ARRAY {
		resTmp := ReadPlainBYTE_ARRAY(bytesReader, cnt)
		res := make([]interface{}, len(resTmp))
		for i := 0; i < len(resTmp); i++ {
			res[i] = resTmp[i]
		}
		return res
	} else if dataType == parquet.Type_FIXED_LEN_BYTE_ARRAY {
		resTmp := ReadPlainFIXED_LEN_BYTE_ARRAY(bytesReader, cnt, bitWidth)
		res := make([]interface{}, len(resTmp))
		for i := 0; i < len(resTmp); i++ {
			res[i] = resTmp[i]
		}
		return res
	} else {
		return nil
	}
}

func ReadPlainINT32(bytesReader *bytes.Reader, cnt uint64) []INT32 {
	res := make([]INT32, cnt)
	for i := 0; i < int(cnt); i++ {
		binary.Read(bytesReader, binary.LittleEndian, &res[i])
	}
	return res
}

func ReadPlainINT64(bytesReader *bytes.Reader, cnt uint64) []INT64 {
	res := make([]INT64, cnt)
	for i := 0; i < int(cnt); i++ {
		binary.Read(bytesReader, binary.LittleEndian, &res[i])
	}
	return res
}

func ReadPlainINT96(bytesReader *bytes.Reader, cnt uint64) []INT96 {
	res := make([]INT96, cnt)
	for i := 0; i < int(cnt); i++ {
		binary.Read(bytesReader, binary.LittleEndian, &res[i])
	}
	return res
}

func ReadPlainFLOAT(bytesReader *bytes.Reader, cnt uint64) []FLOAT {
	res := make([]FLOAT, cnt)
	for i := 0; i < int(cnt); i++ {
		binary.Read(bytesReader, binary.LittleEndian, &res[i])
	}
	return res
}

func ReadPlainDOUBLE(bytesReader *bytes.Reader, cnt uint64) []DOUBLE {
	res := make([]DOUBLE, cnt)
	for i := 0; i < int(cnt); i++ {
		binary.Read(bytesReader, binary.LittleEndian, &res[i])
	}
	return res
}

func ReadPlainBYTE_ARRAY(bytesReader *bytes.Reader, cnt uint64) []BYTE_ARRAY {
	res := make([]BYTE_ARRAY, cnt)
	for i := 0; i < int(cnt); i++ {
		buf := make([]byte, 4)
		bytesReader.Read(buf)
		ln := binary.LittleEndian.Uint32(buf)
		cur := make([]byte, ln)
		bytesReader.Read(cur)
		res[i] = BYTE_ARRAY(cur)
	}
	return res
}

func ReadPlainFIXED_LEN_BYTE_ARRAY(bytesReader *bytes.Reader, cnt uint64, fixedLength uint64) []FIXED_LEN_BYTE_ARRAY {
	res := make([]FIXED_LEN_BYTE_ARRAY, cnt)
	for i := 0; i < int(cnt); i++ {
		cur := make([]byte, fixedLength)
		bytesReader.Read(cur)
		res[i] = FIXED_LEN_BYTE_ARRAY(cur)
	}
	return res
}

func ReadUnsignedVarInt(bytesReader *bytes.Reader) uint64 {
	var res uint64 = 0
	var shift uint64 = 0
	for {
		b, err := bytesReader.ReadByte()
		if err != nil {
			break
		}
		res |= ((uint64(b) & uint64(0x7F)) << uint64(shift))
		if (b & 0x80) == 0 {
			break
		}
		shift += 7
	}
	return res
}

//RLE return res is []INT64
func ReadRLE(bytesReader *bytes.Reader, header uint64, bitWidth uint64) []interface{} {
	cnt := header >> 1
	width := (bitWidth + 7) / 8
	data := make([]byte, width)
	bytesReader.Read(data)
	for len(data) < 4 {
		data = append(data, byte(0))
	}
	val := INT64(binary.LittleEndian.Uint32(data))
	res := make([]interface{}, cnt)

	for i := 0; i < int(cnt); i++ {
		res[i] = val
	}
	return res
}

//return res is []INT64
func ReadBitPacked(bytesReader *bytes.Reader, header uint64, bitWidth uint64) []interface{} {
	numGroup := (header >> 1)
	cnt := numGroup * 8
	byteCnt := cnt * bitWidth / 8
	res := make([]interface{}, 0)
	if bitWidth == 0 {
		for i := 0; i < int(cnt); i++ {
			res = append(res, 0)
		}
		return res
	}
	bytesBuf := make([]byte, byteCnt)
	bytesReader.Read(bytesBuf)

	i := 0
	var resCur uint64 = 0
	var resCurNeedBits uint64 = bitWidth
	var used uint64 = 0
	var left uint64 = 8 - used
	b := bytesBuf[i]
	for i < len(bytesBuf) {
		if left >= resCurNeedBits {
			resCur |= uint64(((uint64(b) >> uint64(used)) & ((1 << uint64(resCurNeedBits)) - 1)) << uint64(bitWidth-resCurNeedBits))
			res = append(res, INT64(resCur))
			left -= resCurNeedBits
			used += resCurNeedBits

			resCurNeedBits = bitWidth
			resCur = 0

			if left <= 0 && i+1 < len(bytesBuf) {
				i += 1
				b = bytesBuf[i]
				left = 8
				used = 0
			}

		} else {
			resCur |= uint64((uint64(b) >> uint64(used)) << uint64(bitWidth-resCurNeedBits))
			i += 1
			if i < len(bytesBuf) {
				b = bytesBuf[i]
			}
			resCurNeedBits -= left
			left = 8
			used = 0
		}
	}
	return res
}

//res is INT64
func ReadRLEBitPackedHybrid(bytesReader *bytes.Reader, bitWidth uint64, length uint64) []interface{} {
	res := make([]interface{}, 0)
	if length <= 0 {
		length = uint64(ReadPlainINT32(bytesReader, 1)[0])
	}
	log.Println("ReadRLEBitPackedHybrid length =", length)

	buf := make([]byte, length)
	bytesReader.Read(buf)
	newReader := bytes.NewReader(buf)
	for newReader.Len() > 0 {
		header := ReadUnsignedVarInt(newReader)
		if header&1 == 0 {
			res = append(res, ReadRLE(newReader, header, bitWidth)...)
		} else {
			res = append(res, ReadBitPacked(newReader, header, bitWidth)...)
		}
	}
	return res
}

//res is INT64
func ReadDeltaINT(bytesReader *bytes.Reader) []interface{} {
	blockSize := ReadUnsignedVarInt(bytesReader)
	numMiniblocksInBlock := ReadUnsignedVarInt(bytesReader)
	numValues := ReadUnsignedVarInt(bytesReader)
	firstValueZigZag := ReadUnsignedVarInt(bytesReader)
	var firstValue int64 = int64(firstValueZigZag>>1) ^ (-int64(firstValueZig & 1))

	res := make([]interface{}, 0)
	for len(res) < numValues {
		minDeltaZigZag := ReadUnsignedVarInt(bytesReader)
		var minDelta int64 = int64(minDeltaZigZag>>1) ^ (-int64(minDeltaZigZag & 1))
		var bitWidths = make([]uint64, numMiniblocksInBlock)
		for i := 0; i < numMiniblocksInBlock; i++ {
			bitWidths[i] = uint64(bytesReader.ReadByte())
		}
	}

	return nil
}
<commit_msg>adding ReadDeltaInt..<commit_after>package PEncoding

import (
	. "ParquetType"
	"bytes"
	"encoding/binary"
	"log"
	"parquet"
)

func ReadPlain(bytesReader *bytes.Reader, dataType parquet.Type, cnt uint64, bitWidth uint64) []interface{} {
	if dataType == parquet.Type_BOOLEAN {
		res := ReadBitPacked(bytesReader, uint64(cnt<<1), 1)
		return res
	} else if dataType == parquet.Type_INT32 {
		resTmp := ReadPlainINT32(bytesReader, cnt)
		res := make([]interface{}, len(resTmp))
		for i := 0; i < len(resTmp); i++ {
			res[i] = resTmp[i]
		}
		return res
	} else if dataType == parquet.Type_INT64 {
		resTmp := ReadPlainINT64(bytesReader, cnt)
		res := make([]interface{}, len(resTmp))
		for i := 0; i < len(resTmp); i++ {
			res[i] = resTmp[i]
		}
		return res
	} else if dataType == parquet.Type_INT96 {
		resTmp := ReadPlainINT96(bytesReader, cnt)
		res := make([]interface{}, len(resTmp))
		for i := 0; i < len(resTmp); i++ {
			res[i] = resTmp[i]
		}
		return res

	} else if dataType == parquet.Type_FLOAT {
		resTmp := ReadPlainFLOAT(bytesReader, cnt)
		res := make([]interface{}, len(resTmp))
		for i := 0; i < len(resTmp); i++ {
			res[i] = resTmp[i]
		}
		return res
	} else if dataType == parquet.Type_DOUBLE {
		resTmp := ReadPlainDOUBLE(bytesReader, cnt)
		res := make([]interface{}, len(resTmp))
		for i := 0; i < len(resTmp); i++ {
			res[i] = resTmp[i]
		}
		return res
	} else if dataType == parquet.Type_BYTE_ARRAY {
		resTmp := ReadPlainBYTE_ARRAY(bytesReader, cnt)
		res := make([]interface{}, len(resTmp))
		for i := 0; i < len(resTmp); i++ {
			res[i] = resTmp[i]
		}
		return res
	} else if dataType == parquet.Type_FIXED_LEN_BYTE_ARRAY {
		resTmp := ReadPlainFIXED_LEN_BYTE_ARRAY(bytesReader, cnt, bitWidth)
		res := make([]interface{}, len(resTmp))
		for i := 0; i < len(resTmp); i++ {
			res[i] = resTmp[i]
		}
		return res
	} else {
		return nil
	}
}

func ReadPlainINT32(bytesReader *bytes.Reader, cnt uint64) []INT32 {
	res := make([]INT32, cnt)
	for i := 0; i < int(cnt); i++ {
		binary.Read(bytesReader, binary.LittleEndian, &res[i])
	}
	return res
}

func ReadPlainINT64(bytesReader *bytes.Reader, cnt uint64) []INT64 {
	res := make([]INT64, cnt)
	for i := 0; i < int(cnt); i++ {
		binary.Read(bytesReader, binary.LittleEndian, &res[i])
	}
	return res
}

func ReadPlainINT96(bytesReader *bytes.Reader, cnt uint64) []INT96 {
	res := make([]INT96, cnt)
	for i := 0; i < int(cnt); i++ {
		binary.Read(bytesReader, binary.LittleEndian, &res[i])
	}
	return res
}

func ReadPlainFLOAT(bytesReader *bytes.Reader, cnt uint64) []FLOAT {
	res := make([]FLOAT, cnt)
	for i := 0; i < int(cnt); i++ {
		binary.Read(bytesReader, binary.LittleEndian, &res[i])
	}
	return res
}

func ReadPlainDOUBLE(bytesReader *bytes.Reader, cnt uint64) []DOUBLE {
	res := make([]DOUBLE, cnt)
	for i := 0; i < int(cnt); i++ {
		binary.Read(bytesReader, binary.LittleEndian, &res[i])
	}
	return res
}

func ReadPlainBYTE_ARRAY(bytesReader *bytes.Reader, cnt uint64) []BYTE_ARRAY {
	res := make([]BYTE_ARRAY, cnt)
	for i := 0; i < int(cnt); i++ {
		buf := make([]byte, 4)
		bytesReader.Read(buf)
		ln := binary.LittleEndian.Uint32(buf)
		cur := make([]byte, ln)
		bytesReader.Read(cur)
		res[i] = BYTE_ARRAY(cur)
	}
	return res
}

func ReadPlainFIXED_LEN_BYTE_ARRAY(bytesReader *bytes.Reader, cnt uint64, fixedLength uint64) []FIXED_LEN_BYTE_ARRAY {
	res := make([]FIXED_LEN_BYTE_ARRAY, cnt)
	for i := 0; i < int(cnt); i++ {
		cur := make([]byte, fixedLength)
		bytesReader.Read(cur)
		res[i] = FIXED_LEN_BYTE_ARRAY(cur)
	}
	return res
}

func ReadUnsignedVarInt(bytesReader *bytes.Reader) uint64 {
	var res uint64 = 0
	var shift uint64 = 0
	for {
		b, err := bytesReader.ReadByte()
		if err != nil {
			break
		}
		res |= ((uint64(b) & uint64(0x7F)) << uint64(shift))
		if (b & 0x80) == 0 {
			break
		}
		shift += 7
	}
	return res
}

//RLE return res is []INT64
func ReadRLE(bytesReader *bytes.Reader, header uint64, bitWidth uint64) []interface{} {
	cnt := header >> 1
	width := (bitWidth + 7) / 8
	data := make([]byte, width)
	bytesReader.Read(data)
	for len(data) < 4 {
		data = append(data, byte(0))
	}
	val := INT64(binary.LittleEndian.Uint32(data))
	res := make([]interface{}, cnt)

	for i := 0; i < int(cnt); i++ {
		res[i] = val
	}
	return res
}

//return res is []INT64
func ReadBitPacked(bytesReader *bytes.Reader, header uint64, bitWidth uint64) []interface{} {
	numGroup := (header >> 1)
	cnt := numGroup * 8
	byteCnt := cnt * bitWidth / 8
	res := make([]interface{}, 0)
	if bitWidth == 0 {
		for i := 0; i < int(cnt); i++ {
			res = append(res, 0)
		}
		return res
	}
	bytesBuf := make([]byte, byteCnt)
	bytesReader.Read(bytesBuf)

	i := 0
	var resCur uint64 = 0
	var resCurNeedBits uint64 = bitWidth
	var used uint64 = 0
	var left uint64 = 8 - used
	b := bytesBuf[i]
	for i < len(bytesBuf) {
		if left >= resCurNeedBits {
			resCur |= uint64(((uint64(b) >> uint64(used)) & ((1 << uint64(resCurNeedBits)) - 1)) << uint64(bitWidth-resCurNeedBits))
			res = append(res, INT64(resCur))
			left -= resCurNeedBits
			used += resCurNeedBits

			resCurNeedBits = bitWidth
			resCur = 0

			if left <= 0 && i+1 < len(bytesBuf) {
				i += 1
				b = bytesBuf[i]
				left = 8
				used = 0
			}

		} else {
			resCur |= uint64((uint64(b) >> uint64(used)) << uint64(bitWidth-resCurNeedBits))
			i += 1
			if i < len(bytesBuf) {
				b = bytesBuf[i]
			}
			resCurNeedBits -= left
			left = 8
			used = 0
		}
	}
	return res
}

//res is INT64
func ReadRLEBitPackedHybrid(bytesReader *bytes.Reader, bitWidth uint64, length uint64) []interface{} {
	res := make([]interface{}, 0)
	if length <= 0 {
		length = uint64(ReadPlainINT32(bytesReader, 1)[0])
	}
	log.Println("ReadRLEBitPackedHybrid length =", length)

	buf := make([]byte, length)
	bytesReader.Read(buf)
	newReader := bytes.NewReader(buf)
	for newReader.Len() > 0 {
		header := ReadUnsignedVarInt(newReader)
		if header&1 == 0 {
			res = append(res, ReadRLE(newReader, header, bitWidth)...)
		} else {
			res = append(res, ReadBitPacked(newReader, header, bitWidth)...)
		}
	}
	return res
}

//res is INT64
func ReadDeltaINT(bytesReader *bytes.Reader) []interface{} {
	blockSize := ReadUnsignedVarInt(bytesReader)
	numMiniblocksInBlock := ReadUnsignedVarInt(bytesReader)
	numValues := ReadUnsignedVarInt(bytesReader)
	firstValueZigZag := ReadUnsignedVarInt(bytesReader)
	var firstValue int64 = int64(firstValueZigZag>>1) ^ (-int64(firstValueZig & 1))

	res := make([]interface{}, 0)
	for len(res) < numValues {
		minDeltaZigZag := ReadUnsignedVarInt(bytesReader)
		var minDelta int64 = int64(minDeltaZigZag>>1) ^ (-int64(minDeltaZigZag & 1))
		var bitWidths = make([]uint64, numMiniblocksInBlock)
		for i := 0; i < numMiniblocksInBlock; i++ {
			bitWidths[i] = uint64(bytesReader.ReadByte())
		}

	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// godoc: Go Documentation Server

// Web server tree:
//
//	http://godoc/		main landing page
//	http://godoc/doc/	serve from $GOROOT/doc - spec, mem, tutorial, etc.
//	http://godoc/src/	serve files from $GOROOT/src; .go gets pretty-printed
//	http://godoc/cmd/	serve documentation about commands (TODO)
//	http://godoc/pkg/	serve documentation about packages
//				(idea is if you say import "compress/zlib", you go to
//				http://godoc/pkg/compress/zlib)
//
// Command-line interface:
//
//	godoc packagepath [name ...]
//
//	godoc compress/zlib
//		- prints doc for package compress/zlib
//	godoc crypto/block Cipher NewCMAC
//		- prints doc for Cipher and NewCMAC in package crypto/block

package main

import (
	"bytes";
	"flag";
	"fmt";
	"http";
	"io";
	"log";
	"os";
	"time";
)

var (
	// periodic sync
	syncCmd				= flag.String("sync", "", "sync command; disabled if empty");
	syncMin				= flag.Int("sync_minutes", 0, "sync interval in minutes; disabled if <= 0");
	syncDelay	delayTime;	// actual sync delay in minutes; usually syncDelay == syncMin, but delay may back off exponentially

	// server control
	httpaddr	= flag.String("http", "", "HTTP service address (e.g., ':6060')");

	// layout control
	html	= flag.Bool("html", false, "print HTML in command-line mode");
)

func exec(c *http.Conn, args []string) (status int) {
	r, w, err := os.Pipe();
	if err != nil {
		log.Stderrf("os.Pipe(): %v\n", err);
		return 2;
	}

	bin := args[0];
	fds := []*os.File{nil, w, w};
	if *verbose {
		log.Stderrf("executing %v", args);
	}
	pid, err := os.ForkExec(bin, args, os.Environ(), goroot, fds);
	defer r.Close();
	w.Close();
	if err != nil {
		log.Stderrf("os.ForkExec(%q): %v\n", bin, err);
		return 2;
	}

	var buf bytes.Buffer;
	io.Copy(&buf, r);
	wait, err := os.Wait(pid, 0);
	if err != nil {
		os.Stderr.Write(buf.Bytes());
		log.Stderrf("os.Wait(%d, 0): %v\n", pid, err);
		return 2;
	}
	status = wait.ExitStatus();
	if !wait.Exited() || status > 1 {
		os.Stderr.Write(buf.Bytes());
		log.Stderrf("executing %v failed (exit status = %d)", args, status);
		return;
	}

	if *verbose {
		os.Stderr.Write(buf.Bytes());
	}
	if c != nil {
		c.SetHeader("content-type", "text/plain; charset=utf-8");
		c.Write(buf.Bytes());
	}

	return;
}

// Maximum directory depth, adjust as needed.
const maxDirDepth = 24

func dosync(c *http.Conn, r *http.Request) {
	args := []string{"/bin/sh", "-c", *syncCmd};
	switch exec(c, args) {
	case 0:
		// sync succeeded and some files have changed;
		// update package tree.
		// TODO(gri): The directory tree may be temporarily out-of-sync.
		//            Consider keeping separate time stamps so the web-
		//            page can indicate this discrepancy.
		fsTree.set(newDirectory(".", maxDirDepth));
		fallthrough;
	case 1:
		// sync failed because no files changed;
		// don't change the package tree
		syncDelay.set(*syncMin);	//  revert to regular sync schedule
	default:
		// sync failed because of an error - back off exponentially, but try at least once a day
		syncDelay.backoff(24*60);
	}
}

func usage() {
	fmt.Fprintf(os.Stderr,
		"usage: godoc package [name ...]\n"
			"	godoc -http=:6060\n");
	flag.PrintDefaults();
	os.Exit(2);
}

func loggingHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(c *http.Conn, req *http.Request) {
		log.Stderrf("%s\t%s", c.RemoteAddr, req.URL);
		h.ServeHTTP(c, req);
	});
}

func main() {
	flag.Usage = usage;
	flag.Parse();

	// Check usage first; get usage message out early.
	switch {
	case *httpaddr != "":
		if flag.NArg() != 0 {
			usage();
		}
	default:
		if flag.NArg() == 0 {
			usage();
		}
	}

	if err := os.Chdir(goroot); err != nil {
		log.Exitf("chdir %s: %v", goroot, err);
	}

	readTemplates();

	if *httpaddr != "" {
		// HTTP server mode.
		var handler http.Handler = http.DefaultServeMux;
		if *verbose {
			log.Stderrf("Go Documentation Server\n");
			log.Stderrf("address = %s\n", *httpaddr);
			log.Stderrf("goroot = %s\n", goroot);
			log.Stderrf("pkgroot = %s\n", *pkgroot);
			log.Stderrf("tmplroot = %s\n", *tmplroot);
			handler = loggingHandler(handler);
		}

		registerPublicHandlers(http.DefaultServeMux);
		if *syncCmd != "" {
			http.Handle("/debug/sync", http.HandlerFunc(dosync));
		}

		// Initialize directory tree with corresponding timestamp.
		// Do it in two steps:
		// 1) set timestamp right away so that the indexer is kicked on
		fsTree.set(nil);
		// 2) compute initial directory tree in a goroutine so that launch is quick
		go func() { fsTree.set(newDirectory(".", maxDirDepth)) }();

		// Start sync goroutine, if enabled.
		if *syncCmd != "" && *syncMin > 0 {
			syncDelay.set(*syncMin);	// initial sync delay
			go func() {
				for {
					dosync(nil, nil);
					delay, _ := syncDelay.get();
					if *verbose {
						log.Stderrf("next sync in %dmin", delay.(int));
					}
					time.Sleep(int64(delay.(int))*60e9);
				}
			}();
		}

		// Start indexing goroutine.
		go indexer();

		// The server may have been restarted; always wait 1sec to
		// give the forking server a chance to shut down and release
		// the http port.
		// TODO(gri): Do we still need this?
		time.Sleep(1e9);

		// Start http server.
		if err := http.ListenAndServe(*httpaddr, handler); err != nil {
			log.Exitf("ListenAndServe %s: %v", *httpaddr, err);
		}
		return;
	}

	// Command line mode.
	if *html {
		packageText = packageHTML;
		parseerrorText = parseerrorHTML;
	}

	info := pkgHandler.getPageInfo(flag.Arg(0));

	if info.PDoc == nil && info.Dirs == nil {
		// try again, this time assume it's a command
		info = cmdHandler.getPageInfo(flag.Arg(0));
	}

	if info.PDoc != nil && flag.NArg() > 1 {
		args := flag.Args();
		info.PDoc.Filter(args[1:len(args)]);
	}

	if err := packageText.Execute(info, os.Stdout); err != nil {
		log.Stderrf("packageText.Execute: %s", err);
	}
}
<commit_msg>- exit godoc w/o crashing if negative tabwidth is provided - minor related cleanups<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// godoc: Go Documentation Server

// Web server tree:
//
//	http://godoc/		main landing page
//	http://godoc/doc/	serve from $GOROOT/doc - spec, mem, tutorial, etc.
//	http://godoc/src/	serve files from $GOROOT/src; .go gets pretty-printed
//	http://godoc/cmd/	serve documentation about commands (TODO)
//	http://godoc/pkg/	serve documentation about packages
//				(idea is if you say import "compress/zlib", you go to
//				http://godoc/pkg/compress/zlib)
//
// Command-line interface:
//
//	godoc packagepath [name ...]
//
//	godoc compress/zlib
//		- prints doc for package compress/zlib
//	godoc crypto/block Cipher NewCMAC
//		- prints doc for Cipher and NewCMAC in package crypto/block

package main

import (
	"bytes";
	"flag";
	"fmt";
	"http";
	"io";
	"log";
	"os";
	"time";
)

var (
	// periodic sync
	syncCmd				= flag.String("sync", "", "sync command; disabled if empty");
	syncMin				= flag.Int("sync_minutes", 0, "sync interval in minutes; disabled if <= 0");
	syncDelay	delayTime;	// actual sync delay in minutes; usually syncDelay == syncMin, but delay may back off exponentially

	// server control
	httpaddr	= flag.String("http", "", "HTTP service address (e.g., ':6060')");

	// layout control
	html	= flag.Bool("html", false, "print HTML in command-line mode");
)

func exec(c *http.Conn, args []string) (status int) {
	r, w, err := os.Pipe();
	if err != nil {
		log.Stderrf("os.Pipe(): %v\n", err);
		return 2;
	}

	bin := args[0];
	fds := []*os.File{nil, w, w};
	if *verbose {
		log.Stderrf("executing %v", args);
	}
	pid, err := os.ForkExec(bin, args, os.Environ(), goroot, fds);
	defer r.Close();
	w.Close();
	if err != nil {
		log.Stderrf("os.ForkExec(%q): %v\n", bin, err);
		return 2;
	}

	var buf bytes.Buffer;
	io.Copy(&buf, r);
	wait, err := os.Wait(pid, 0);
	if err != nil {
		os.Stderr.Write(buf.Bytes());
		log.Stderrf("os.Wait(%d, 0): %v\n", pid, err);
		return 2;
	}
	status = wait.ExitStatus();
	if !wait.Exited() || status > 1 {
		os.Stderr.Write(buf.Bytes());
		log.Stderrf("executing %v failed (exit status = %d)", args, status);
		return;
	}

	if *verbose {
		os.Stderr.Write(buf.Bytes());
	}
	if c != nil {
		c.SetHeader("content-type", "text/plain; charset=utf-8");
		c.Write(buf.Bytes());
	}

	return;
}

// Maximum directory depth, adjust as needed.
const maxDirDepth = 24

func dosync(c *http.Conn, r *http.Request) {
	args := []string{"/bin/sh", "-c", *syncCmd};
	switch exec(c, args) {
	case 0:
		// sync succeeded and some files have changed;
		// update package tree.
		// TODO(gri): The directory tree may be temporarily out-of-sync.
		//            Consider keeping separate time stamps so the web-
		//            page can indicate this discrepancy.
		fsTree.set(newDirectory(".", maxDirDepth));
		fallthrough;
	case 1:
		// sync failed because no files changed;
		// don't change the package tree
		syncDelay.set(*syncMin);	//  revert to regular sync schedule
	default:
		// sync failed because of an error - back off exponentially, but try at least once a day
		syncDelay.backoff(24*60);
	}
}

func usage() {
	fmt.Fprintf(os.Stderr,
		"usage: godoc package [name ...]\n"
			"	godoc -http=:6060\n");
	flag.PrintDefaults();
	os.Exit(2);
}

func loggingHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(c *http.Conn, req *http.Request) {
		log.Stderrf("%s\t%s", c.RemoteAddr, req.URL);
		h.ServeHTTP(c, req);
	});
}

func main() {
	flag.Usage = usage;
	flag.Parse();

	// Check usage: either server and no args, or command line and args
	if (*httpaddr != "") != (flag.NArg() == 0) {
		usage();
	}

	if *tabwidth < 0 {
		log.Exitf("negative tabwidth %d", *tabwidth);
	}

	if err := os.Chdir(goroot); err != nil {
		log.Exitf("chdir %s: %v", goroot, err);
	}

	readTemplates();

	if *httpaddr != "" {
		// HTTP server mode.
		var handler http.Handler = http.DefaultServeMux;
		if *verbose {
			log.Stderrf("Go Documentation Server\n");
			log.Stderrf("address = %s\n", *httpaddr);
			log.Stderrf("goroot = %s\n", goroot);
			log.Stderrf("cmdroot = %s\n", *cmdroot);
			log.Stderrf("pkgroot = %s\n", *pkgroot);
			log.Stderrf("tmplroot = %s\n", *tmplroot);
			log.Stderrf("tabwidth = %d\n", *tabwidth);
			handler = loggingHandler(handler);
		}

		registerPublicHandlers(http.DefaultServeMux);
		if *syncCmd != "" {
			http.Handle("/debug/sync", http.HandlerFunc(dosync));
		}

		// Initialize directory tree with corresponding timestamp.
		// Do it in two steps:
		// 1) set timestamp right away so that the indexer is kicked on
		fsTree.set(nil);
		// 2) compute initial directory tree in a goroutine so that launch is quick
		go func() { fsTree.set(newDirectory(".", maxDirDepth)) }();

		// Start sync goroutine, if enabled.
		if *syncCmd != "" && *syncMin > 0 {
			syncDelay.set(*syncMin);	// initial sync delay
			go func() {
				for {
					dosync(nil, nil);
					delay, _ := syncDelay.get();
					if *verbose {
						log.Stderrf("next sync in %dmin", delay.(int));
					}
					time.Sleep(int64(delay.(int))*60e9);
				}
			}();
		}

		// Start indexing goroutine.
		go indexer();

		// The server may have been restarted; always wait 1sec to
		// give the forking server a chance to shut down and release
		// the http port.
		// TODO(gri): Do we still need this?
		time.Sleep(1e9);

		// Start http server.
		if err := http.ListenAndServe(*httpaddr, handler); err != nil {
			log.Exitf("ListenAndServe %s: %v", *httpaddr, err);
		}
		return;
	}

	// Command line mode.
	if *html {
		packageText = packageHTML;
		parseerrorText = parseerrorHTML;
	}

	info := pkgHandler.getPageInfo(flag.Arg(0));

	if info.PDoc == nil && info.Dirs == nil {
		// try again, this time assume it's a command
		info = cmdHandler.getPageInfo(flag.Arg(0));
	}

	if info.PDoc != nil && flag.NArg() > 1 {
		args := flag.Args();
		info.PDoc.Filter(args[1:len(args)]);
	}

	if err := packageText.Execute(info, os.Stdout); err != nil {
		log.Stderrf("packageText.Execute: %s", err);
	}
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/moby/src/initrd"
)

// Process the build arguments and execute build
func build(args []string) {
	buildCmd := flag.NewFlagSet("build", flag.ExitOnError)
	buildCmd.Usage = func() {
		fmt.Printf("USAGE: %s build [options] <file>[.yml]\n\n", os.Args[0])
		fmt.Printf("Options:\n")
		buildCmd.PrintDefaults()
	}
	buildName := buildCmd.String("name", "", "Name to use for output files")
	buildPull := buildCmd.Bool("pull", false, "Always pull images")

	buildCmd.Parse(args)
	remArgs := buildCmd.Args()

	if len(remArgs) == 0 {
		fmt.Println("Please specify a configuration file")
		buildCmd.Usage()
		os.Exit(1)
	}
	conf := remArgs[0]
	if filepath.Ext(conf) == "" {
		conf = conf + ".yml"
	}

	buildInternal(*buildName, *buildPull, conf)
}

// Perform the actual build process
func buildInternal(name string, pull bool, conf string) {
	if name == "" {
		name = filepath.Base(conf)
		ext := filepath.Ext(conf)
		if ext != "" {
			name = name[:len(name)-len(ext)]
		}
	}

	config, err := ioutil.ReadFile(conf)
	if err != nil {
		log.Fatalf("Cannot open config file: %v", err)
	}

	m, err := NewConfig(config)
	if err != nil {
		log.Fatalf("Invalid config: %v", err)
	}

	containers := []*bytes.Buffer{}

	if pull {
		log.Infof("Pull kernel image: %s", m.Kernel.Image)
		err := dockerPull(m.Kernel.Image)
		if err != nil {
			log.Fatalf("Could not pull image %s: %v", m.Kernel.Image, err)
		}
	}
	// get kernel bzImage and initrd tarball from container
	// TODO examine contents to see what names they might have
	log.Infof("Extract kernel image: %s", m.Kernel.Image)
	const (
		bzimageName = "bzImage"
		ktarName    = "kernel.tar"
	)
	out, err := dockerRun(m.Kernel.Image, "tar", "cf", "-", bzimageName, ktarName)
	if err != nil {
		log.Fatalf("Failed to extract kernel image and tarball: %v", err)
	}
	buf := bytes.NewBuffer(out)
	bzimage, ktar, err := untarKernel(buf, bzimageName, ktarName)
	if err != nil {
		log.Fatalf("Could not extract bzImage and kernel filesystem from tarball. %v", err)
	}
	containers = append(containers, ktar)

	// convert init image to tarball
	if pull {
		log.Infof("Pull init: %s", m.Init)
		err := dockerPull(m.Init)
		if err != nil {
			log.Fatalf("Could not pull image %s: %v", m.Init, err)
		}
	}
	log.Infof("Process init: %s", m.Init)
	init, err := ImageExtract(m.Init, "")
	if err != nil {
		log.Fatalf("Failed to build init tarball: %v", err)
	}
	buffer := bytes.NewBuffer(init)
	containers = append(containers, buffer)

	log.Infof("Add system containers:")
	for i, image := range m.System {
		if pull {
			log.Infof("  Pull: %s", image.Image)
			err := dockerPull(image.Image)
			if err != nil {
				log.Fatalf("Could not pull image %s: %v", image.Image, err)
			}
		}
		log.Infof("  Create OCI config for %s", image.Image)
		config, err := ConfigToOCI(&image)
		if err != nil {
			log.Fatalf("Failed to create config.json for %s: %v", image.Image, err)
		}
		so := fmt.Sprintf("%03d", i)
		path := "containers/system/" + so + "-" + image.Name
		out, err := ImageBundle(path, image.Image, config)
		if err != nil {
			log.Fatalf("Failed to extract root filesystem for %s: %v", image.Image, err)
		}
		buffer := bytes.NewBuffer(out)
		containers = append(containers, buffer)
	}

	log.Infof("Add daemon containers:")
	for _, image := range m.Daemon {
		if pull {
			log.Infof("  Pull: %s", image.Image)
			err := dockerPull(image.Image)
			if err != nil {
				log.Fatalf("Could not pull image %s: %v", image.Image, err)
			}
		}
		log.Infof("  Create OCI config for %s", image.Image)
		config, err := ConfigToOCI(&image)
		if err != nil {
			log.Fatalf("Failed to create config.json for %s: %v", image.Image, err)
		}
		path := "containers/daemon/" + image.Name
		out, err := ImageBundle(path, image.Image, config)
		if err != nil {
			log.Fatalf("Failed to extract root filesystem for %s: %v", image.Image, err)
		}
		buffer := bytes.NewBuffer(out)
		containers = append(containers, buffer)
	}

	// add files
	buffer, err = filesystem(m)
	if err != nil {
		log.Fatalf("failed to add filesystem parts: %v", err)
	}
	containers = append(containers, buffer)

	log.Infof("Create initial ram disk")
	initrd, err := containersInitrd(containers)
	if err != nil {
		log.Fatalf("Failed to make initrd %v", err)
	}

	log.Infof("Create outputs:")
	err = outputs(m, name, bzimage.Bytes(), initrd.Bytes())
	if err != nil {
		log.Fatalf("Error writing outputs: %v", err)
	}
}

func untarKernel(buf *bytes.Buffer, bzimageName, ktarName string) (*bytes.Buffer, *bytes.Buffer, error) {
	tr := tar.NewReader(buf)

	var bzimage, ktar *bytes.Buffer

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatalln(err)
		}
		switch hdr.Name {
		case bzimageName:
			bzimage = new(bytes.Buffer)
			_, err := io.Copy(bzimage, tr)
			if err != nil {
				return nil, nil, err
			}
		case ktarName:
			ktar = new(bytes.Buffer)
			_, err := io.Copy(ktar, tr)
			if err != nil {
				return nil, nil, err
			}
		default:
			continue
		}
	}

	if ktar == nil || bzimage == nil {
		return nil, nil, errors.New("did not find bzImage and kernel.tar in tarball")
	}

	return bzimage, ktar, nil
}

func containersInitrd(containers []*bytes.Buffer) (*bytes.Buffer, error) {
	w := new(bytes.Buffer)
	iw := initrd.NewWriter(w)
	defer iw.Close()
	for _, file := range containers {
		_, err := initrd.Copy(iw, file)
		if err != nil {
			return nil, err
		}
	}

	return w, nil
}
<commit_msg>cli: Fix "build" when the basename contains a "."<commit_after>package main

import (
	"archive/tar"
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/moby/src/initrd"
)

// Process the build arguments and execute build
func build(args []string) {
	buildCmd := flag.NewFlagSet("build", flag.ExitOnError)
	buildCmd.Usage = func() {
		fmt.Printf("USAGE: %s build [options] <file>[.yml]\n\n", os.Args[0])
		fmt.Printf("Options:\n")
		buildCmd.PrintDefaults()
	}
	buildName := buildCmd.String("name", "", "Name to use for output files")
	buildPull := buildCmd.Bool("pull", false, "Always pull images")

	buildCmd.Parse(args)
	remArgs := buildCmd.Args()

	if len(remArgs) == 0 {
		fmt.Println("Please specify a configuration file")
		buildCmd.Usage()
		os.Exit(1)
	}
	conf := remArgs[0]
	if !(filepath.Ext(conf) == ".yml" || filepath.Ext(conf) == ".yaml") {
		conf = conf + ".yml"
	}

	buildInternal(*buildName, *buildPull, conf)
}

// Perform the actual build process
func buildInternal(name string, pull bool, conf string) {
	if name == "" {
		name = filepath.Base(conf)
		ext := filepath.Ext(conf)
		if ext != "" {
			name = name[:len(name)-len(ext)]
		}
	}

	config, err := ioutil.ReadFile(conf)
	if err != nil {
		log.Fatalf("Cannot open config file: %v", err)
	}

	m, err := NewConfig(config)
	if err != nil {
		log.Fatalf("Invalid config: %v", err)
	}

	containers := []*bytes.Buffer{}

	if pull {
		log.Infof("Pull kernel image: %s", m.Kernel.Image)
		err := dockerPull(m.Kernel.Image)
		if err != nil {
			log.Fatalf("Could not pull image %s: %v", m.Kernel.Image, err)
		}
	}
	// get kernel bzImage and initrd tarball from container
	// TODO examine contents to see what names they might have
	log.Infof("Extract kernel image: %s", m.Kernel.Image)
	const (
		bzimageName = "bzImage"
		ktarName    = "kernel.tar"
	)
	out, err := dockerRun(m.Kernel.Image, "tar", "cf", "-", bzimageName, ktarName)
	if err != nil {
		log.Fatalf("Failed to extract kernel image and tarball: %v", err)
	}
	buf := bytes.NewBuffer(out)
	bzimage, ktar, err := untarKernel(buf, bzimageName, ktarName)
	if err != nil {
		log.Fatalf("Could not extract bzImage and kernel filesystem from tarball. %v", err)
	}
	containers = append(containers, ktar)

	// convert init image to tarball
	if pull {
		log.Infof("Pull init: %s", m.Init)
		err := dockerPull(m.Init)
		if err != nil {
			log.Fatalf("Could not pull image %s: %v", m.Init, err)
		}
	}
	log.Infof("Process init: %s", m.Init)
	init, err := ImageExtract(m.Init, "")
	if err != nil {
		log.Fatalf("Failed to build init tarball: %v", err)
	}
	buffer := bytes.NewBuffer(init)
	containers = append(containers, buffer)

	log.Infof("Add system containers:")
	for i, image := range m.System {
		if pull {
			log.Infof("  Pull: %s", image.Image)
			err := dockerPull(image.Image)
			if err != nil {
				log.Fatalf("Could not pull image %s: %v", image.Image, err)
			}
		}
		log.Infof("  Create OCI config for %s", image.Image)
		config, err := ConfigToOCI(&image)
		if err != nil {
			log.Fatalf("Failed to create config.json for %s: %v", image.Image, err)
		}
		so := fmt.Sprintf("%03d", i)
		path := "containers/system/" + so + "-" + image.Name
		out, err := ImageBundle(path, image.Image, config)
		if err != nil {
			log.Fatalf("Failed to extract root filesystem for %s: %v", image.Image, err)
		}
		buffer := bytes.NewBuffer(out)
		containers = append(containers, buffer)
	}

	log.Infof("Add daemon containers:")
	for _, image := range m.Daemon {
		if pull {
			log.Infof("  Pull: %s", image.Image)
			err := dockerPull(image.Image)
			if err != nil {
				log.Fatalf("Could not pull image %s: %v", image.Image, err)
			}
		}
		log.Infof("  Create OCI config for %s", image.Image)
		config, err := ConfigToOCI(&image)
		if err != nil {
			log.Fatalf("Failed to create config.json for %s: %v", image.Image, err)
		}
		path := "containers/daemon/" + image.Name
		out, err := ImageBundle(path, image.Image, config)
		if err != nil {
			log.Fatalf("Failed to extract root filesystem for %s: %v", image.Image, err)
		}
		buffer := bytes.NewBuffer(out)
		containers = append(containers, buffer)
	}

	// add files
	buffer, err = filesystem(m)
	if err != nil {
		log.Fatalf("failed to add filesystem parts: %v", err)
	}
	containers = append(containers, buffer)

	log.Infof("Create initial ram disk")
	initrd, err := containersInitrd(containers)
	if err != nil {
		log.Fatalf("Failed to make initrd %v", err)
	}

	log.Infof("Create outputs:")
	err = outputs(m, name, bzimage.Bytes(), initrd.Bytes())
	if err != nil {
		log.Fatalf("Error writing outputs: %v", err)
	}
}

func untarKernel(buf *bytes.Buffer, bzimageName, ktarName string) (*bytes.Buffer, *bytes.Buffer, error) {
	tr := tar.NewReader(buf)

	var bzimage, ktar *bytes.Buffer

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatalln(err)
		}
		switch hdr.Name {
		case bzimageName:
			bzimage = new(bytes.Buffer)
			_, err := io.Copy(bzimage, tr)
			if err != nil {
				return nil, nil, err
			}
		case ktarName:
			ktar = new(bytes.Buffer)
			_, err := io.Copy(ktar, tr)
			if err != nil {
				return nil, nil, err
			}
		default:
			continue
		}
	}

	if ktar == nil || bzimage == nil {
		return nil, nil, errors.New("did not find bzImage and kernel.tar in tarball")
	}

	return bzimage, ktar, nil
}

func containersInitrd(containers []*bytes.Buffer) (*bytes.Buffer, error) {
	w := new(bytes.Buffer)
	iw := initrd.NewWriter(w)
	defer iw.Close()
	for _, file := range containers {
		_, err := initrd.Copy(iw, file)
		if err != nil {
			return nil, err
		}
	}

	return w, nil
}
<|endoftext|>


<commit_before>package net

import (
	"io"
	"log"
	"net"
	"sync"

	"../redlot"
)

var counter struct {
	sync.RWMutex
	ConnCounter uint64
	TotalCalls  uint64
}

func init() {
	// Register commands.
	// system info
	REG("INFO", StatusReply, info)

	// KV type
	REG("GET", BulkReply, redlot.Get)
	REG("SET", StatusReply, redlot.Set)
	REG("DEL", StatusReply, redlot.Del)
	REG("EXISTS", IntReply, redlot.Exists)
	REG("SETX", StatusReply, redlot.Setx)
	REG("SETEX", StatusReply, redlot.Setx) // Alias of SETX
	REG("TTL", IntReply, redlot.TTL)
	REG("EXPIRE", IntReply, redlot.Expire)
	REGL("KEYS", ListReply, redlot.Keys)
	REGL("RKEYS", ListReply, redlot.Rkeys)
	REGL("SCAN", ListReply, redlot.Scan)
	REGL("RSCAN", ListReply, redlot.Rscan)
	REGL("MULTI_GET", ListReply, redlot.MultiGet)
	REG("MULTI_SET", StatusReply, redlot.MultiSet)
}

func Serve(addr string, options *redlot.Options) {
	// Open LevelDB with options.
	redlot.Open(options)

	// Create sockets listener.
	l, err := net.Listen("tcp4", addr)
	if err != nil {
		log.Fatalf("Listen error: %v\n", err.Error())
	}
	defer l.Close()

	for {
		conn, err := l.Accept()
		if err != nil {
			log.Fatalf("Wait for a connection error: %s\n", err.Error())
		}

		// Count connecion
		counter.Lock()
		counter.ConnCounter++
		counter.Unlock()

		go func(c net.Conn) {
			for {
				req, err := newRequset(c)
				if err == io.EOF {
					break
				}
				if err != nil {
					continue
				}

				counter.Lock()
				counter.TotalCalls++
				counter.Unlock()

				r := run(req.Cmd, req.Args)
				r.WriteTo(c)
			}

			c.Close()
			counter.Lock()
			counter.ConnCounter--
			counter.Unlock()

		}(conn)

	}
}
<commit_msg>Register INCR command.<commit_after>package net

import (
	"io"
	"log"
	"net"
	"sync"

	"../redlot"
)

var counter struct {
	sync.RWMutex
	ConnCounter uint64
	TotalCalls  uint64
}

func init() {
	// Register commands.
	// system info
	REG("INFO", StatusReply, info)

	// KV type
	REG("GET", BulkReply, redlot.Get)
	REG("SET", StatusReply, redlot.Set)
	REG("INCR", IntReply, redlot.Incr)
	REG("DEL", StatusReply, redlot.Del)
	REG("EXISTS", IntReply, redlot.Exists)
	REG("SETX", StatusReply, redlot.Setx)
	REG("SETEX", StatusReply, redlot.Setx) // Alias of SETX
	REG("TTL", IntReply, redlot.TTL)
	REG("EXPIRE", IntReply, redlot.Expire)
	REGL("KEYS", ListReply, redlot.Keys)
	REGL("RKEYS", ListReply, redlot.Rkeys)
	REGL("SCAN", ListReply, redlot.Scan)
	REGL("RSCAN", ListReply, redlot.Rscan)
	REGL("MULTI_GET", ListReply, redlot.MultiGet)
	REG("MULTI_SET", StatusReply, redlot.MultiSet)
}

func Serve(addr string, options *redlot.Options) {
	// Open LevelDB with options.
	redlot.Open(options)

	// Create sockets listener.
	l, err := net.Listen("tcp4", addr)
	if err != nil {
		log.Fatalf("Listen error: %v\n", err.Error())
	}
	defer l.Close()

	for {
		conn, err := l.Accept()
		if err != nil {
			log.Fatalf("Wait for a connection error: %s\n", err.Error())
		}

		// Count connecion
		counter.Lock()
		counter.ConnCounter++
		counter.Unlock()

		go func(c net.Conn) {
			for {
				req, err := newRequset(c)
				if err == io.EOF {
					break
				}
				if err != nil {
					continue
				}

				counter.Lock()
				counter.TotalCalls++
				counter.Unlock()

				r := run(req.Cmd, req.Args)
				r.WriteTo(c)
			}

			c.Close()
			counter.Lock()
			counter.ConnCounter--
			counter.Unlock()

		}(conn)

	}
}
<|endoftext|>


<commit_before>package get

import (
	"compress/gzip"
	"crypto"
	"encoding/xml"
	"io"
	"log"

	"github.com/moio/minima/util"
)

// common

// XMLLocation maps a <location> tag in repodata/repomd.xml or repodata/<ID>-primary.xml.gz
type XMLLocation struct {
	Href string `xml:"href,attr"`
}

// repodata/repomd.xml

// XMLRepomd maps a <repomd> tag in repodata/repomd.xml
type XMLRepomd struct {
	Data []XMLData `xml:"data"`
}

// XMLData maps a <data> tag in repodata/repomd.xml
type XMLData struct {
	Type     string      `xml:"type,attr"`
	Location XMLLocation `xml:"location"`
}

// repodata/<ID>-primary.xml.gz

// XMLMetaData maps a <metadata> tag in repodata/<ID>-primary.xml.gz
type XMLMetaData struct {
	Packages []XMLPackage `xml:"package"`
}

// XMLPackage maps a <package> tag in repodata/<ID>-primary.xml.gz
type XMLPackage struct {
	Arch     string      `xml:"arch"`
	Location XMLLocation `xml:"location"`
	Checksum XMLChecksum `xml:"checksum"`
}

// XMLChecksum maps a <checksum> tag in repodata/<ID>-primary.xml.gz
type XMLChecksum struct {
	Type     string `xml:"type,attr"`
	Checksum string `xml:",cdata"`
}

var hashMap = map[string]crypto.Hash{
	"sha":    crypto.SHA1,
	"sha1":   crypto.SHA1,
	"sha256": crypto.SHA256,
}

const repomdPath = "repodata/repomd.xml"

// Syncer syncs repos from an HTTP source to a Storage
type Syncer struct {
	// URL of the repo this syncer syncs
	Url     string
	archs   map[string]bool
	storage Storage
}

// NewSyncer creates a new Syncer
func NewSyncer(url string, archs map[string]bool, storage Storage) *Syncer {
	return &Syncer{url, archs, storage}
}

// StoreRepo stores an HTTP repo in a Storage, automatically retrying in case of recoverable errors
func (r *Syncer) StoreRepo() (err error) {
	checksumMap := r.readChecksumMap()
	for i := 0; i < 10; i++ {
		err = r.storeRepo(checksumMap)
		if err == nil {
			return
		}

		uerr, unexpectedStatusCode := err.(*UnexpectedStatusCodeError)
		if unexpectedStatusCode {
			if uerr.StatusCode == 404 {
				log.Printf("Got 404, presumably temporarily, retrying...\n")
			} else {
				return err
			}
		}

		_, checksumError := err.(*util.ChecksumError)
		if checksumError {
			log.Printf("Checksum did not match, presumably the repo was published while syncing, retrying...\n")
		} else {
			return err
		}
	}

	log.Printf("Too many temporary errors, aborting...\n")
	return err
}

// StoreRepo stores an HTTP repo in a Storage
func (r *Syncer) storeRepo(checksumMap map[string]XMLChecksum) (err error) {
	packagesToDownload, packagesToRecycle, err := r.processMetadata(checksumMap)
	if err != nil {
		return
	}

	downloadCount := len(packagesToDownload)
	log.Printf("Downloading %v packages...\n", downloadCount)
	for _, pack := range packagesToDownload {
		err = r.downloadStoreApply(pack.Location.Href, pack.Checksum.Checksum, hashMap[pack.Checksum.Type], util.Nop)
		if err != nil {
			return err
		}
	}

	recycleCount := len(packagesToRecycle)
	log.Printf("Recycling %v packages...\n", recycleCount)
	for _, pack := range packagesToRecycle {
		err = r.storage.Recycle(pack.Location.Href)
		if err != nil {
			return
		}
	}

	log.Printf("Committing changes...\n")
	err = r.storage.Commit()
	if err != nil {
		return
	}
	return
}

// downloadStore downloads a repo-relative path into a file
func (r *Syncer) downloadStore(path string) error {
	return r.downloadStoreApply(path, "", 0, util.Nop)
}

// downloadStoreApply downloads a repo-relative path into a file, while applying a ReaderConsumer
func (r *Syncer) downloadStoreApply(path string, checksum string, hash crypto.Hash, f util.ReaderConsumer) error {
	log.Printf("Downloading %v...", path)
	body, err := ReadURL(r.Url + "/" + path)
	if err != nil {
		return err
	}

	return util.Compose(r.storage.StoringMapper(path, checksum, hash), f)(body)
}

// processMetadata stores the repo metadata and returns a list of package file
// paths to download
func (r *Syncer) processMetadata(checksumMap map[string]XMLChecksum) (packagesToDownload []XMLPackage, packagesToRecycle []XMLPackage, err error) {
	err = r.downloadStoreApply(repomdPath, "", 0, func(reader io.ReadCloser) (err error) {
		decoder := xml.NewDecoder(reader)
		var repomd XMLRepomd
		err = decoder.Decode(&repomd)
		if err != nil {
			return
		}

		data := repomd.Data
		for i := 0; i < len(data); i++ {
			metadataPath := data[i].Location.Href
			if data[i].Type == "primary" {
				packagesToDownload, packagesToRecycle, err = r.processPrimary(metadataPath, checksumMap)
			} else {
				err = r.downloadStore(metadataPath)
			}
			if err != nil {
				return
			}
		}
		return
	})
	if err != nil {
		return
	}

	err = r.downloadStore(repomdPath + ".asc")
	if err != nil {
		uerr, unexpectedStatusCode := err.(*UnexpectedStatusCodeError)
		if unexpectedStatusCode && uerr.StatusCode == 404 {
			log.Printf("Got 404, ignoring...")
		} else {
			return
		}
	}

	err = r.downloadStore(repomdPath + ".key")
	if err != nil {
		uerr, unexpectedStatusCode := err.(*UnexpectedStatusCodeError)
		if unexpectedStatusCode && uerr.StatusCode == 404 {
			log.Printf("Got 404, ignoring...")
			err = nil
		} else {
			return
		}
	}

	return
}

func (r *Syncer) readMetaData(reader io.Reader) (primary XMLMetaData, err error) {
	gzReader, err := gzip.NewReader(reader)
	if err != nil {
		return
	}
	defer gzReader.Close()

	decoder := xml.NewDecoder(gzReader)
	err = decoder.Decode(&primary)

	return
}

func (r *Syncer) readChecksumMap() (checksumMap map[string]XMLChecksum) {
	checksumMap = make(map[string]XMLChecksum)
	repomdReader, err := r.storage.NewReader(repomdPath, Permanent)
	if err != nil {
		if err == ErrFileNotFound {
			log.Println("First-time sync started")
		} else {
			log.Println(err.Error())
			log.Println("Error while reading previously-downloaded metadata. Starting sync from scratch")
		}
		return
	}
	defer repomdReader.Close()

	decoder := xml.NewDecoder(repomdReader)
	var repomd XMLRepomd
	err = decoder.Decode(&repomd)
	if err != nil {
		log.Println(err.Error())
		log.Println("Error while parsing previously-downloaded metadata. Starting sync from scratch")
		return
	}

	data := repomd.Data
	for i := 0; i < len(data); i++ {
		metadataPath := data[i].Location.Href
		if data[i].Type == "primary" {
			primaryReader, err := r.storage.NewReader(metadataPath, Permanent)
			if err != nil {
				return
			}
			primary, err := r.readMetaData(primaryReader)
			if err != nil {
				return
			}
			for _, pack := range primary.Packages {
				checksumMap[pack.Location.Href] = pack.Checksum
			}
		}
	}
	return
}

// processPrimary stores the primary XML metadata file and returns a list of
// package file paths to download
func (r *Syncer) processPrimary(path string, checksumMap map[string]XMLChecksum) (packagesToDownload []XMLPackage, packagesToRecycle []XMLPackage, err error) {
	err = r.downloadStoreApply(path, "", 0, func(reader io.ReadCloser) (err error) {
		primary, err := r.readMetaData(reader)
		if err != nil {
			return
		}

		allArchs := len(r.archs) == 0
		for _, pack := range primary.Packages {
			if allArchs || pack.Arch == "noarch" || r.archs[pack.Arch] {
				previousChecksum, foundInPermanentLocation := checksumMap[pack.Location.Href]
				if !foundInPermanentLocation || previousChecksum.Type != pack.Checksum.Type || previousChecksum.Checksum != pack.Checksum.Checksum {
					reader, err := r.storage.NewReader(pack.Location.Href, Temporary)
					if err != nil {
						log.Printf("...package '%v' not found or not recyclable, will be downloaded\n", pack.Location.Href)
						packagesToDownload = append(packagesToDownload, pack)
					} else {
						checksum, err := util.Checksum(reader, hashMap[pack.Checksum.Type])
						if err != nil || checksum != pack.Checksum.Checksum {
							log.Printf("...package '%v' found in partially-downloaded repo, not recyclable, will be re-downloaded\n", pack.Location.Href)
							packagesToDownload = append(packagesToDownload, pack)
						} else {
							log.Printf("...package '%v' found in partially-downloaded repo, recyclable, will be skipped\n", pack.Location.Href)
						}
					}
				} else {
					log.Printf("...package '%v' found in already-downloaded repo, recyclable, will be recycled\n", pack.Location.Href)
					packagesToRecycle = append(packagesToRecycle, pack)
				}
			}
		}
		return
	})
	return
}
<commit_msg>Add metadata checksum to checksumMap<commit_after>package get

import (
	"compress/gzip"
	"crypto"
	"encoding/xml"
	"io"
	"log"

	"github.com/moio/minima/util"
)

// common

// XMLLocation maps a <location> tag in repodata/repomd.xml or repodata/<ID>-primary.xml.gz
type XMLLocation struct {
	Href string `xml:"href,attr"`
}

// repodata/repomd.xml

// XMLRepomd maps a <repomd> tag in repodata/repomd.xml
type XMLRepomd struct {
	Data []XMLData `xml:"data"`
}

// XMLData maps a <data> tag in repodata/repomd.xml
type XMLData struct {
	Type     string      `xml:"type,attr"`
	Location XMLLocation `xml:"location"`
	Checksum XMLChecksum `xml:"checksum"`
}

// repodata/<ID>-primary.xml.gz

// XMLMetaData maps a <metadata> tag in repodata/<ID>-primary.xml.gz
type XMLMetaData struct {
	Packages []XMLPackage `xml:"package"`
}

// XMLPackage maps a <package> tag in repodata/<ID>-primary.xml.gz
type XMLPackage struct {
	Arch     string      `xml:"arch"`
	Location XMLLocation `xml:"location"`
	Checksum XMLChecksum `xml:"checksum"`
}

// XMLChecksum maps a <checksum> tag in repodata/<ID>-primary.xml.gz
type XMLChecksum struct {
	Type     string `xml:"type,attr"`
	Checksum string `xml:",cdata"`
}

var hashMap = map[string]crypto.Hash{
	"sha":    crypto.SHA1,
	"sha1":   crypto.SHA1,
	"sha256": crypto.SHA256,
}

const repomdPath = "repodata/repomd.xml"

// Syncer syncs repos from an HTTP source to a Storage
type Syncer struct {
	// URL of the repo this syncer syncs
	Url     string
	archs   map[string]bool
	storage Storage
}

// NewSyncer creates a new Syncer
func NewSyncer(url string, archs map[string]bool, storage Storage) *Syncer {
	return &Syncer{url, archs, storage}
}

// StoreRepo stores an HTTP repo in a Storage, automatically retrying in case of recoverable errors
func (r *Syncer) StoreRepo() (err error) {
	checksumMap := r.readChecksumMap()
	for i := 0; i < 10; i++ {
		err = r.storeRepo(checksumMap)
		if err == nil {
			return
		}

		uerr, unexpectedStatusCode := err.(*UnexpectedStatusCodeError)
		if unexpectedStatusCode {
			if uerr.StatusCode == 404 {
				log.Printf("Got 404, presumably temporarily, retrying...\n")
			} else {
				return err
			}
		}

		_, checksumError := err.(*util.ChecksumError)
		if checksumError {
			log.Printf("Checksum did not match, presumably the repo was published while syncing, retrying...\n")
		} else {
			return err
		}
	}

	log.Printf("Too many temporary errors, aborting...\n")
	return err
}

// StoreRepo stores an HTTP repo in a Storage
func (r *Syncer) storeRepo(checksumMap map[string]XMLChecksum) (err error) {
	packagesToDownload, packagesToRecycle, err := r.processMetadata(checksumMap)
	if err != nil {
		return
	}

	downloadCount := len(packagesToDownload)
	log.Printf("Downloading %v packages...\n", downloadCount)
	for _, pack := range packagesToDownload {
		err = r.downloadStoreApply(pack.Location.Href, pack.Checksum.Checksum, hashMap[pack.Checksum.Type], util.Nop)
		if err != nil {
			return err
		}
	}

	recycleCount := len(packagesToRecycle)
	log.Printf("Recycling %v packages...\n", recycleCount)
	for _, pack := range packagesToRecycle {
		err = r.storage.Recycle(pack.Location.Href)
		if err != nil {
			return
		}
	}

	log.Printf("Committing changes...\n")
	err = r.storage.Commit()
	if err != nil {
		return
	}
	return
}

// downloadStore downloads a repo-relative path into a file
func (r *Syncer) downloadStore(path string) error {
	return r.downloadStoreApply(path, "", 0, util.Nop)
}

// downloadStoreApply downloads a repo-relative path into a file, while applying a ReaderConsumer
func (r *Syncer) downloadStoreApply(path string, checksum string, hash crypto.Hash, f util.ReaderConsumer) error {
	log.Printf("Downloading %v...", path)
	body, err := ReadURL(r.Url + "/" + path)
	if err != nil {
		return err
	}

	return util.Compose(r.storage.StoringMapper(path, checksum, hash), f)(body)
}

// processMetadata stores the repo metadata and returns a list of package file
// paths to download
func (r *Syncer) processMetadata(checksumMap map[string]XMLChecksum) (packagesToDownload []XMLPackage, packagesToRecycle []XMLPackage, err error) {
	err = r.downloadStoreApply(repomdPath, "", 0, func(reader io.ReadCloser) (err error) {
		decoder := xml.NewDecoder(reader)
		var repomd XMLRepomd
		err = decoder.Decode(&repomd)
		if err != nil {
			return
		}

		data := repomd.Data
		for i := 0; i < len(data); i++ {
			metadataPath := data[i].Location.Href
			if data[i].Type == "primary" {
				packagesToDownload, packagesToRecycle, err = r.processPrimary(metadataPath, checksumMap)
			} else {
				err = r.downloadStore(metadataPath)
			}
			if err != nil {
				return
			}
		}
		return
	})
	if err != nil {
		return
	}

	err = r.downloadStore(repomdPath + ".asc")
	if err != nil {
		uerr, unexpectedStatusCode := err.(*UnexpectedStatusCodeError)
		if unexpectedStatusCode && uerr.StatusCode == 404 {
			log.Printf("Got 404, ignoring...")
		} else {
			return
		}
	}

	err = r.downloadStore(repomdPath + ".key")
	if err != nil {
		uerr, unexpectedStatusCode := err.(*UnexpectedStatusCodeError)
		if unexpectedStatusCode && uerr.StatusCode == 404 {
			log.Printf("Got 404, ignoring...")
			err = nil
		} else {
			return
		}
	}

	return
}

func (r *Syncer) readMetaData(reader io.Reader) (primary XMLMetaData, err error) {
	gzReader, err := gzip.NewReader(reader)
	if err != nil {
		return
	}
	defer gzReader.Close()

	decoder := xml.NewDecoder(gzReader)
	err = decoder.Decode(&primary)

	return
}

func (r *Syncer) readChecksumMap() (checksumMap map[string]XMLChecksum) {
	checksumMap = make(map[string]XMLChecksum)
	repomdReader, err := r.storage.NewReader(repomdPath, Permanent)
	if err != nil {
		if err == ErrFileNotFound {
			log.Println("First-time sync started")
		} else {
			log.Println(err.Error())
			log.Println("Error while reading previously-downloaded metadata. Starting sync from scratch")
		}
		return
	}
	defer repomdReader.Close()

	decoder := xml.NewDecoder(repomdReader)
	var repomd XMLRepomd
	err = decoder.Decode(&repomd)
	if err != nil {
		log.Println(err.Error())
		log.Println("Error while parsing previously-downloaded metadata. Starting sync from scratch")
		return
	}

	data := repomd.Data
	for i := 0; i < len(data); i++ {
		dataHref := data[i].Location.Href
		dataChecksum := data[i].Checksum
		checksumMap[dataHref] = dataChecksum
		if data[i].Type == "primary" {
			primaryReader, err := r.storage.NewReader(dataHref, Permanent)
			if err != nil {
				return
			}
			primary, err := r.readMetaData(primaryReader)
			if err != nil {
				return
			}
			for _, pack := range primary.Packages {
				checksumMap[pack.Location.Href] = pack.Checksum
			}
		}
	}
	return
}

// processPrimary stores the primary XML metadata file and returns a list of
// package file paths to download
func (r *Syncer) processPrimary(path string, checksumMap map[string]XMLChecksum) (packagesToDownload []XMLPackage, packagesToRecycle []XMLPackage, err error) {
	err = r.downloadStoreApply(path, "", 0, func(reader io.ReadCloser) (err error) {
		primary, err := r.readMetaData(reader)
		if err != nil {
			return
		}

		allArchs := len(r.archs) == 0
		for _, pack := range primary.Packages {
			if allArchs || pack.Arch == "noarch" || r.archs[pack.Arch] {
				previousChecksum, foundInPermanentLocation := checksumMap[pack.Location.Href]
				if !foundInPermanentLocation || previousChecksum.Type != pack.Checksum.Type || previousChecksum.Checksum != pack.Checksum.Checksum {
					reader, err := r.storage.NewReader(pack.Location.Href, Temporary)
					if err != nil {
						log.Printf("...package '%v' not found or not recyclable, will be downloaded\n", pack.Location.Href)
						packagesToDownload = append(packagesToDownload, pack)
					} else {
						checksum, err := util.Checksum(reader, hashMap[pack.Checksum.Type])
						if err != nil || checksum != pack.Checksum.Checksum {
							log.Printf("...package '%v' found in partially-downloaded repo, not recyclable, will be re-downloaded\n", pack.Location.Href)
							packagesToDownload = append(packagesToDownload, pack)
						} else {
							log.Printf("...package '%v' found in partially-downloaded repo, recyclable, will be skipped\n", pack.Location.Href)
						}
					}
				} else {
					log.Printf("...package '%v' found in already-downloaded repo, recyclable, will be recycled\n", pack.Location.Href)
					packagesToRecycle = append(packagesToRecycle, pack)
				}
			}
		}
		return
	})
	return
}
<|endoftext|>


<commit_before>package rootfs_provider

import (
	"net/url"
	"sync"
	"time"

	"github.com/docker/docker/daemon/graphdriver"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"

	"github.com/cloudfoundry-incubator/garden-linux/old/repository_fetcher"
	"github.com/cloudfoundry-incubator/garden-linux/process"
)

type dockerRootFSProvider struct {
	graphDriver   graphdriver.Driver
	volumeCreator VolumeCreator
	repoFetcher   repository_fetcher.RepositoryFetcher
	namespacer    Namespacer
	clock         clock.Clock
	cleaner       Cleaner
	mutex         *sync.Mutex

	fallback RootFSProvider
}

//go:generate counterfeiter -o fake_graph_driver/fake_graph_driver.go . GraphDriver
type GraphDriver interface {
	graphdriver.Driver
}

//go:generate counterfeiter -o fake_cleaner/fake_cleaner.go . Cleaner
type Cleaner interface {
	Clean(id string) error
}

type NoopCleaner struct{}

func (NoopCleaner) Clean(id string) error { return nil }

func NewDocker(
	repoFetcher repository_fetcher.RepositoryFetcher,
	graphDriver GraphDriver,
	volumeCreator VolumeCreator,
	namespacer Namespacer,
	clock clock.Clock,
	cleaner Cleaner,
) (RootFSProvider, error) {
	return &dockerRootFSProvider{
		repoFetcher:   repoFetcher,
		graphDriver:   graphDriver,
		volumeCreator: volumeCreator,
		namespacer:    namespacer,
		clock:         clock,
		cleaner:       cleaner,
		mutex:         &sync.Mutex{},
	}, nil
}

func (provider *dockerRootFSProvider) ProvideRootFS(logger lager.Logger, id string, url *url.URL, shouldNamespace bool) (string, process.Env, error) {
	tag := "latest"
	if len(url.Fragment) > 0 {
		tag = url.Fragment
	}

	imageID, envvars, volumes, err := provider.repoFetcher.Fetch(logger, url, tag)
	if err != nil {
		return "", nil, err
	}

	if shouldNamespace {
		provider.mutex.Lock()
		imageID, err = provider.namespace(imageID)
		provider.mutex.Unlock()
		if err != nil {
			return "", nil, err
		}
	}

	err = provider.graphDriver.Create(id, imageID)
	if err != nil {
		return "", nil, err
	}

	rootPath, err := provider.graphDriver.Get(id, "")
	if err != nil {
		return "", nil, err
	}

	for _, v := range volumes {
		if err = provider.volumeCreator.Create(rootPath, v); err != nil {
			return "", nil, err
		}
	}

	return rootPath, envvars, nil
}

func (provider *dockerRootFSProvider) namespace(imageID string) (string, error) {
	namespacedImageID := imageID + "@namespaced"
	if !provider.graphDriver.Exists(namespacedImageID) {
		if err := provider.createNamespacedLayer(namespacedImageID, imageID); err != nil {
			return "", err
		}
	}

	return namespacedImageID, nil
}

func (provider *dockerRootFSProvider) createNamespacedLayer(id string, parentId string) error {
	var err error
	var path string
	if path, err = provider.createLayer(id, parentId); err != nil {
		return err
	}

	return provider.namespacer.Namespace(path)
}

func (provider *dockerRootFSProvider) createLayer(id, parentId string) (string, error) {
	errs := func(err error) (string, error) {
		return "", err
	}

	if err := provider.graphDriver.Create(id, parentId); err != nil {
		return errs(err)
	}

	namespacedRootfs, err := provider.graphDriver.Get(id, "")
	if err != nil {
		return errs(err)
	}

	return namespacedRootfs, nil
}

func (provider *dockerRootFSProvider) CleanupRootFS(logger lager.Logger, id string) error {
	provider.graphDriver.Put(id)

	if err := provider.cleaner.Clean(id); err != nil {
		return err
	}

	var err error
	maxAttempts := 10

	for errorCount := 0; errorCount < maxAttempts; errorCount++ {
		err = provider.graphDriver.Remove(id)
		if err == nil {
			break
		}

		logger.Error("cleanup-rootfs", err, lager.Data{
			"current-attempts": errorCount + 1,
			"max-attempts":     maxAttempts,
		})

		provider.clock.Sleep(200 * time.Millisecond)
	}

	return nil
}
<commit_msg>Avoid swallowing error message if maxRetries is exceeded<commit_after>package rootfs_provider

import (
	"net/url"
	"sync"
	"time"

	"github.com/docker/docker/daemon/graphdriver"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"

	"github.com/cloudfoundry-incubator/garden-linux/old/repository_fetcher"
	"github.com/cloudfoundry-incubator/garden-linux/process"
)

type dockerRootFSProvider struct {
	graphDriver   graphdriver.Driver
	volumeCreator VolumeCreator
	repoFetcher   repository_fetcher.RepositoryFetcher
	namespacer    Namespacer
	clock         clock.Clock
	cleaner       Cleaner
	mutex         *sync.Mutex

	fallback RootFSProvider
}

//go:generate counterfeiter -o fake_graph_driver/fake_graph_driver.go . GraphDriver
type GraphDriver interface {
	graphdriver.Driver
}

//go:generate counterfeiter -o fake_cleaner/fake_cleaner.go . Cleaner
type Cleaner interface {
	Clean(id string) error
}

type NoopCleaner struct{}

func (NoopCleaner) Clean(id string) error { return nil }

func NewDocker(
	repoFetcher repository_fetcher.RepositoryFetcher,
	graphDriver GraphDriver,
	volumeCreator VolumeCreator,
	namespacer Namespacer,
	clock clock.Clock,
	cleaner Cleaner,
) (RootFSProvider, error) {
	return &dockerRootFSProvider{
		repoFetcher:   repoFetcher,
		graphDriver:   graphDriver,
		volumeCreator: volumeCreator,
		namespacer:    namespacer,
		clock:         clock,
		cleaner:       cleaner,
		mutex:         &sync.Mutex{},
	}, nil
}

func (provider *dockerRootFSProvider) ProvideRootFS(logger lager.Logger, id string, url *url.URL, shouldNamespace bool) (string, process.Env, error) {
	tag := "latest"
	if len(url.Fragment) > 0 {
		tag = url.Fragment
	}

	imageID, envvars, volumes, err := provider.repoFetcher.Fetch(logger, url, tag)
	if err != nil {
		return "", nil, err
	}

	if shouldNamespace {
		provider.mutex.Lock()
		imageID, err = provider.namespace(imageID)
		provider.mutex.Unlock()
		if err != nil {
			return "", nil, err
		}
	}

	err = provider.graphDriver.Create(id, imageID)
	if err != nil {
		return "", nil, err
	}

	rootPath, err := provider.graphDriver.Get(id, "")
	if err != nil {
		return "", nil, err
	}

	for _, v := range volumes {
		if err = provider.volumeCreator.Create(rootPath, v); err != nil {
			return "", nil, err
		}
	}

	return rootPath, envvars, nil
}

func (provider *dockerRootFSProvider) namespace(imageID string) (string, error) {
	namespacedImageID := imageID + "@namespaced"
	if !provider.graphDriver.Exists(namespacedImageID) {
		if err := provider.createNamespacedLayer(namespacedImageID, imageID); err != nil {
			return "", err
		}
	}

	return namespacedImageID, nil
}

func (provider *dockerRootFSProvider) createNamespacedLayer(id string, parentId string) error {
	var err error
	var path string
	if path, err = provider.createLayer(id, parentId); err != nil {
		return err
	}

	return provider.namespacer.Namespace(path)
}

func (provider *dockerRootFSProvider) createLayer(id, parentId string) (string, error) {
	errs := func(err error) (string, error) {
		return "", err
	}

	if err := provider.graphDriver.Create(id, parentId); err != nil {
		return errs(err)
	}

	namespacedRootfs, err := provider.graphDriver.Get(id, "")
	if err != nil {
		return errs(err)
	}

	return namespacedRootfs, nil
}

func (provider *dockerRootFSProvider) CleanupRootFS(logger lager.Logger, id string) error {
	provider.graphDriver.Put(id)

	if err := provider.cleaner.Clean(id); err != nil {
		return err
	}

	var err error
	maxAttempts := 10
	for errorCount := 0; errorCount < maxAttempts; errorCount++ {
		err = provider.graphDriver.Remove(id)
		if err == nil {
			return nil
		}

		logger.Error("cleanup-rootfs", err, lager.Data{
			"current-attempts": errorCount + 1,
			"max-attempts":     maxAttempts,
		})

		provider.clock.Sleep(200 * time.Millisecond)
	}

	return err
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package gitiles is a client library for the Gitiles source viewer.
package gitiles

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"path"

	"github.com/google/slothfs/cookie"
	"golang.org/x/net/context"
	"golang.org/x/time/rate"
)

// Service is a client for the Gitiles JSON interface.
type Service struct {
	limiter *rate.Limiter
	addr    url.URL
	client  http.Client
	agent   string
	jar     http.CookieJar
}

// Addr returns the address of the gitiles service.
func (s *Service) Addr() string {
	return s.addr.String()
}

// Options configures the the Gitiles service.
type Options struct {
	// A URL for the Gitiles service.
	Address string

	BurstQPS     int
	SustainedQPS float64

	// Path to a Netscape/Mozilla style cookie file.
	CookieJar string

	// UserAgent defines how we present ourself to the server.
	UserAgent string
}

var defaultOptions Options

// DefineFlags sets up standard command line flags, and returns the
// options struct in which the values are put.
func DefineFlags() *Options {
	flag.StringVar(&defaultOptions.Address, "gitiles_url", "https://android.googlesource.com", "URL of the gitiles service.")
	flag.StringVar(&defaultOptions.CookieJar, "gitiles_cookies", "", "path to cURL-style cookie jar file.")
	flag.StringVar(&defaultOptions.UserAgent, "gitiles_agent", "slothfs", "gitiles User-Agent string to use.")
	flag.IntVar(&defaultOptions.BurstQPS, "gitiles_qps", 4, "maximum Gitiles QPS")
	return &defaultOptions
}

// NewService returns a new Gitiles JSON client.
func NewService(opts Options) (*Service, error) {
	var jar http.CookieJar
	if nm := opts.CookieJar; nm != "" {
		var err error
		jar, err = cookie.NewJar(nm)
		if err != nil {
			return nil, err
		}
		if err := cookie.WatchJar(jar, nm); err != nil {
			return nil, err
		}
	}

	if opts.BurstQPS == 0 {
		opts.BurstQPS = 4
	}
	if opts.SustainedQPS == 0.0 {
		opts.SustainedQPS = 0.5
	}

	url, err := url.Parse(opts.Address)
	if err != nil {
		return nil, err
	}
	s := &Service{
		limiter: rate.NewLimiter(rate.Limit(opts.SustainedQPS), opts.BurstQPS),
		addr:    *url,
		agent:   opts.UserAgent,
	}

	s.client.Jar = jar
	s.client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
		req.Header.Set("User-Agent", s.agent)
		return nil
	}
	return s, nil
}

func (s *Service) get(u *url.URL) ([]byte, error) {
	ctx := context.Background()

	if err := s.limiter.Wait(ctx); err != nil {
		return nil, err
	}
	req, err := http.NewRequest("GET", u.String(), nil)
	if err != nil {
		return nil, err
	}
	req.Header.Add("User-Agent", s.agent)
	resp, err := s.client.Do(req)

	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("%s: %s", u.String(), resp.Status)
	}
	if got := resp.Request.URL.String(); got != u.String() {
		// We accept redirects, but only for authentication.
		// If we get a 200 from a different page than we
		// requested, it's probably some sort of login page.
		return nil, fmt.Errorf("got URL %s, want %s", got, u.String())
	}

	c, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	if resp.Header.Get("Content-Type") == "text/plain; charset=UTF-8" {
		out := make([]byte, base64.StdEncoding.DecodedLen(len(c)))
		n, err := base64.StdEncoding.Decode(out, c)
		return out[:n], err
	}
	return c, nil
}

var xssTag = []byte(")]}'\n")

func (s *Service) getJSON(u *url.URL, dest interface{}) error {
	c, err := s.get(u)
	if err != nil {
		return err
	}

	if !bytes.HasPrefix(c, xssTag) {
		return fmt.Errorf("Gitiles JSON %s missing XSS tag: %q", u, c)
	}
	c = c[len(xssTag):]

	err = json.Unmarshal(c, dest)
	if err != nil {
		err = fmt.Errorf("Unmarshal(%s): %v", u, err)
	}
	return err
}

// List retrieves the list of projects.
func (s *Service) List(branches []string) (map[string]*Project, error) {
	listURL := s.addr
	listURL.RawQuery = "format=JSON"
	for _, b := range branches {
		listURL.RawQuery += "&b=" + b
	}

	projects := map[string]*Project{}
	err := s.getJSON(&listURL, &projects)
	return projects, err
}

// NewRepoService creates a service for a specific repository on a Gitiles server.
func (s *Service) NewRepoService(name string) *RepoService {
	return &RepoService{
		Name:    name,
		service: s,
	}
}

// RepoService is a JSON client for the functionality of a specific
// respository.
type RepoService struct {
	Name    string
	service *Service
}

// Get retrieves a single project.
func (s *RepoService) Get() (*Project, error) {
	jsonURL := s.service.addr
	jsonURL.Path = path.Join(jsonURL.Path, s.Name)
	jsonURL.RawQuery = "format=JSON"

	var p Project
	err := s.service.getJSON(&jsonURL, &p)
	return &p, err
}

// GetBlob fetches a blob.
func (s *RepoService) GetBlob(branch, filename string) ([]byte, error) {
	blobURL := s.service.addr

	blobURL.Path = path.Join(blobURL.Path, s.Name, "+show", branch, filename)
	blobURL.RawQuery = "format=TEXT"

	// TODO(hanwen): invent a more structured mechanism for logging.
	log.Println(blobURL.String())
	return s.service.get(&blobURL)
}

// GetTree fetches a tree. The dir argument may not point to a
// blob. If recursive is given, the server recursively expands the
// tree.
func (s *RepoService) GetTree(branch, dir string, recursive bool) (*Tree, error) {
	jsonURL := s.service.addr
	jsonURL.Path = path.Join(jsonURL.Path, s.Name, "+", branch, dir)
	if dir == "" {
		jsonURL.Path += "/"
	}
	jsonURL.RawQuery = "format=JSON&long=1"

	if recursive {
		jsonURL.RawQuery += "&recursive=1"
	}

	var tree Tree
	err := s.service.getJSON(&jsonURL, &tree)
	return &tree, err
}

// GetCommit gets the data of a commit in a branch.
func (s *RepoService) GetCommit(branch string) (*Commit, error) {
	jsonURL := s.service.addr
	jsonURL.Path = path.Join(jsonURL.Path, s.Name, "+", branch)
	jsonURL.RawQuery = "format=JSON"

	var c Commit
	err := s.service.getJSON(&jsonURL, &c)
	return &c, err
}
<commit_msg>Use SustainedQPS for --gitiles_qps flag.<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package gitiles is a client library for the Gitiles source viewer.
package gitiles

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"path"

	"github.com/google/slothfs/cookie"
	"golang.org/x/net/context"
	"golang.org/x/time/rate"
)

// Service is a client for the Gitiles JSON interface.
type Service struct {
	limiter *rate.Limiter
	addr    url.URL
	client  http.Client
	agent   string
	jar     http.CookieJar
}

// Addr returns the address of the gitiles service.
func (s *Service) Addr() string {
	return s.addr.String()
}

// Options configures the the Gitiles service.
type Options struct {
	// A URL for the Gitiles service.
	Address string

	BurstQPS     int
	SustainedQPS float64

	// Path to a Netscape/Mozilla style cookie file.
	CookieJar string

	// UserAgent defines how we present ourself to the server.
	UserAgent string
}

var defaultOptions Options

// DefineFlags sets up standard command line flags, and returns the
// options struct in which the values are put.
func DefineFlags() *Options {
	flag.StringVar(&defaultOptions.Address, "gitiles_url", "https://android.googlesource.com", "URL of the gitiles service.")
	flag.StringVar(&defaultOptions.CookieJar, "gitiles_cookies", "", "path to cURL-style cookie jar file.")
	flag.StringVar(&defaultOptions.UserAgent, "gitiles_agent", "slothfs", "gitiles User-Agent string to use.")
	flag.IntVar(&defaultOptions.SustainedQPS, "gitiles_qps", 4, "maximum Gitiles QPS")
	return &defaultOptions
}

// NewService returns a new Gitiles JSON client.
func NewService(opts Options) (*Service, error) {
	var jar http.CookieJar
	if nm := opts.CookieJar; nm != "" {
		var err error
		jar, err = cookie.NewJar(nm)
		if err != nil {
			return nil, err
		}
		if err := cookie.WatchJar(jar, nm); err != nil {
			return nil, err
		}
	}

	if opts.BurstQPS == 0 {
		opts.BurstQPS = 4
	}
	if opts.SustainedQPS == 0.0 {
		opts.SustainedQPS = 0.5
	}

	url, err := url.Parse(opts.Address)
	if err != nil {
		return nil, err
	}
	s := &Service{
		limiter: rate.NewLimiter(rate.Limit(opts.SustainedQPS), opts.BurstQPS),
		addr:    *url,
		agent:   opts.UserAgent,
	}

	s.client.Jar = jar
	s.client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
		req.Header.Set("User-Agent", s.agent)
		return nil
	}
	return s, nil
}

func (s *Service) get(u *url.URL) ([]byte, error) {
	ctx := context.Background()

	if err := s.limiter.Wait(ctx); err != nil {
		return nil, err
	}
	req, err := http.NewRequest("GET", u.String(), nil)
	if err != nil {
		return nil, err
	}
	req.Header.Add("User-Agent", s.agent)
	resp, err := s.client.Do(req)

	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("%s: %s", u.String(), resp.Status)
	}
	if got := resp.Request.URL.String(); got != u.String() {
		// We accept redirects, but only for authentication.
		// If we get a 200 from a different page than we
		// requested, it's probably some sort of login page.
		return nil, fmt.Errorf("got URL %s, want %s", got, u.String())
	}

	c, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	if resp.Header.Get("Content-Type") == "text/plain; charset=UTF-8" {
		out := make([]byte, base64.StdEncoding.DecodedLen(len(c)))
		n, err := base64.StdEncoding.Decode(out, c)
		return out[:n], err
	}
	return c, nil
}

var xssTag = []byte(")]}'\n")

func (s *Service) getJSON(u *url.URL, dest interface{}) error {
	c, err := s.get(u)
	if err != nil {
		return err
	}

	if !bytes.HasPrefix(c, xssTag) {
		return fmt.Errorf("Gitiles JSON %s missing XSS tag: %q", u, c)
	}
	c = c[len(xssTag):]

	err = json.Unmarshal(c, dest)
	if err != nil {
		err = fmt.Errorf("Unmarshal(%s): %v", u, err)
	}
	return err
}

// List retrieves the list of projects.
func (s *Service) List(branches []string) (map[string]*Project, error) {
	listURL := s.addr
	listURL.RawQuery = "format=JSON"
	for _, b := range branches {
		listURL.RawQuery += "&b=" + b
	}

	projects := map[string]*Project{}
	err := s.getJSON(&listURL, &projects)
	return projects, err
}

// NewRepoService creates a service for a specific repository on a Gitiles server.
func (s *Service) NewRepoService(name string) *RepoService {
	return &RepoService{
		Name:    name,
		service: s,
	}
}

// RepoService is a JSON client for the functionality of a specific
// respository.
type RepoService struct {
	Name    string
	service *Service
}

// Get retrieves a single project.
func (s *RepoService) Get() (*Project, error) {
	jsonURL := s.service.addr
	jsonURL.Path = path.Join(jsonURL.Path, s.Name)
	jsonURL.RawQuery = "format=JSON"

	var p Project
	err := s.service.getJSON(&jsonURL, &p)
	return &p, err
}

// GetBlob fetches a blob.
func (s *RepoService) GetBlob(branch, filename string) ([]byte, error) {
	blobURL := s.service.addr

	blobURL.Path = path.Join(blobURL.Path, s.Name, "+show", branch, filename)
	blobURL.RawQuery = "format=TEXT"

	// TODO(hanwen): invent a more structured mechanism for logging.
	log.Println(blobURL.String())
	return s.service.get(&blobURL)
}

// GetTree fetches a tree. The dir argument may not point to a
// blob. If recursive is given, the server recursively expands the
// tree.
func (s *RepoService) GetTree(branch, dir string, recursive bool) (*Tree, error) {
	jsonURL := s.service.addr
	jsonURL.Path = path.Join(jsonURL.Path, s.Name, "+", branch, dir)
	if dir == "" {
		jsonURL.Path += "/"
	}
	jsonURL.RawQuery = "format=JSON&long=1"

	if recursive {
		jsonURL.RawQuery += "&recursive=1"
	}

	var tree Tree
	err := s.service.getJSON(&jsonURL, &tree)
	return &tree, err
}

// GetCommit gets the data of a commit in a branch.
func (s *RepoService) GetCommit(branch string) (*Commit, error) {
	jsonURL := s.service.addr
	jsonURL.Path = path.Join(jsonURL.Path, s.Name, "+", branch)
	jsonURL.RawQuery = "format=JSON"

	var c Commit
	err := s.service.getJSON(&jsonURL, &c)
	return &c, err
}
<|endoftext|>