<commit_before>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gles

import (
	"context"
	"fmt"
	"strings"

	"github.com/google/gapid/core/data/binary"
	"github.com/google/gapid/core/log"
	"github.com/google/gapid/core/os/device"
	"github.com/google/gapid/core/text"
	"github.com/google/gapid/gapis/api"
	"github.com/google/gapid/gapis/api/transform"
	"github.com/google/gapid/gapis/capture"
	"github.com/google/gapid/gapis/memory"
	"github.com/google/gapid/gapis/replay"
	"github.com/google/gapid/gapis/replay/builder"
	"github.com/google/gapid/gapis/replay/value"
	"github.com/google/gapid/gapis/service"
	"github.com/google/gapid/gapis/shadertools"
)

// findIssues is a command transform that detects issues when replaying the
// stream of commands. Any issues that are found are written to all the chans in
// the slice out. Once the last issue is sent (if any) all the chans in out are
// closed.
type findIssues struct {
	state         *api.GlobalState
	device        *device.Instance
	targetVersion *Version
	issues        []replay.Issue
	res           []replay.Result
	lastGlError   GLenum
}

func newFindIssues(ctx context.Context, c *capture.Capture, device *device.Instance) *findIssues {
	targetVersion, _ := ParseVersion(device.Configuration.Drivers.Opengl.Version)
	transform := &findIssues{
		state:         c.NewState(ctx),
		device:        device,
		targetVersion: targetVersion,
	}
	transform.state.OnError = func(err interface{}) {
		if glenum, ok := err.(GLenum); ok {
			transform.lastGlError = glenum
		}
	}
	return transform
}

// reportTo adds the chan c to the list of issue listeners.
func (t *findIssues) reportTo(r replay.Result) { t.res = append(t.res, r) }

func (t *findIssues) onIssue(cmd api.Cmd, id api.CmdID, s service.Severity, e error) {
	if s == service.Severity_FatalLevel && isIssueWhitelisted(cmd, e) {
		s = service.Severity_ErrorLevel
	}
	t.issues = append(t.issues, replay.Issue{Command: id, Severity: s, Error: e})
}

// The value 0 is used for many enums - prefer GL_NO_ERROR in this case.
func (e GLenum) ErrorString() string {
	if e == GLenum_GL_NO_ERROR {
		return "GL_NO_ERROR"
	}
	return e.String()
}

type ErrUnexpectedDriverTraceError struct {
	DriverError   GLenum
	ExpectedError GLenum
}

func (e ErrUnexpectedDriverTraceError) Error() string {
	return fmt.Sprintf("%s in trace driver, but we expected %s",
		e.DriverError.ErrorString(), e.ExpectedError.ErrorString())
}

func (t *findIssues) Transform(ctx context.Context, id api.CmdID, cmd api.Cmd, out transform.Writer) {
	ctx = log.Enter(ctx, "findIssues")
	cb := CommandBuilder{Thread: cmd.Thread(), Arena: t.state.Arena}
	t.lastGlError = GLenum_GL_NO_ERROR
	mutateErr := cmd.Mutate(ctx, id, t.state, nil /* no builder */, nil /* no watcher */)

	mutatorsGlError := t.lastGlError
	if e := FindErrorState(cmd.Extras()); e != nil {
		// Check that our API file agrees with the driver which we used for tracking.
		if (e.TraceDriversGlError != GLenum_GL_NO_ERROR) != (mutatorsGlError != GLenum_GL_NO_ERROR) {
			errorMsg := ErrUnexpectedDriverTraceError{
				DriverError:   e.TraceDriversGlError,
				ExpectedError: mutatorsGlError,
			}
			t.onIssue(cmd, id, service.Severity_FatalLevel, errorMsg)
		}
		// Check that the C++ and Go versions of the generated code precisely agree.
		if e.InterceptorsGlError != mutatorsGlError {
			t.onIssue(cmd, id, service.Severity_FatalLevel, fmt.Errorf("%s in interceptor, but we expected %s",
				e.InterceptorsGlError.ErrorString(), mutatorsGlError.ErrorString()))
		}
	}

	if mutateErr != nil {
		// Ignore since downstream transform layers can only consume valid commands
		return
	}

	out.MutateAndWrite(ctx, id, cmd)

	dID := id.Derived()
	s := GetState(t.state)
	c := s.GetContext(cmd.Thread())
	if c.IsNil() {
		return
	}

	// Check the result of glGetError after every command.
	out.MutateAndWrite(ctx, dID, cb.Custom(func(ctx context.Context, s *api.GlobalState, b *builder.Builder) error {
		ptr := b.AllocateTemporaryMemory(4)
		b.Call(funcInfoGlGetError)
		b.Store(ptr)
		b.Post(ptr, 4, builder.Postback(func(r binary.Reader, err error) {
			if err != nil {
				t.onIssue(cmd, id, service.Severity_FatalLevel, fmt.Errorf("Failed to decode glGetError postback: %v", err))
				return
			}
			v := GLenum(r.Uint32())
			err = r.Error()
			if err != nil {
				t.onIssue(cmd, id, service.Severity_FatalLevel, fmt.Errorf("Failed to decode glGetError postback: %v", err))
				return
			}
			if v != GLenum_GL_NO_ERROR {
				t.onIssue(cmd, id, service.Severity_FatalLevel, fmt.Errorf("%v in replay driver", v))
			}
		}))
		return nil
	}))

	// null-terminated byte slice to string
	ntbs := func(b []byte) string {
		s := string(b)
		for i, r := range s {
			if r == 0 {
				return strings.TrimSpace(s[:i])
			}
		}
		return strings.TrimSpace(s)
	}

	switch cmd := cmd.(type) {
	case *GlCompileShader:
		shader := c.Objects().Shaders().Get(cmd.Shader())
		st, err := shader.Type().ShaderType()
		if err != nil {
			t.onIssue(cmd, id, service.Severity_ErrorLevel, err)
			return
		}

		if t.targetVersion.IsES {
			// Check we are able to convert this GLES shader to desktop GL.
			opts := shadertools.ConvertOptions{
				ShaderType:        st,
				CheckAfterChanges: true,
				Disassemble:       true,
				TargetGLSLVersion: 430,
			}

			if _, err := shadertools.ConvertGlsl(shader.Source(), &opts); err != nil {
				t.onIssue(cmd, id, service.Severity_ErrorLevel, err)
			}
		}

		const buflen = 8192
		tmp := t.state.AllocOrPanic(ctx, buflen)

		infoLog := make([]byte, buflen)
		out.MutateAndWrite(ctx, dID, cb.GlGetShaderInfoLog(cmd.Shader(), buflen, memory.Nullptr, tmp.Ptr()))
		out.MutateAndWrite(ctx, dID, cb.Custom(func(ctx context.Context, s *api.GlobalState, b *builder.Builder) error {
			b.ReserveMemory(tmp.Range())
			b.Post(value.ObservedPointer(tmp.Address()), buflen, func(r binary.Reader, err error) {
				if err != nil {
					t.onIssue(cmd, id, service.Severity_FatalLevel, fmt.Errorf("Failed to decode glGetShaderInfoLog postback: %v", err))
					return
				}
				r.Data(infoLog)
				if r.Error() != nil {
					t.onIssue(cmd, id, service.Severity_FatalLevel, fmt.Errorf("Failed to decode glGetShaderInfoLog postback: %v", r.Error()))
				}
			})
			return nil
		}))

		source := make([]byte, buflen)
		out.MutateAndWrite(ctx, dID, cb.GlGetShaderSource(cmd.Shader(), buflen, memory.Nullptr, tmp.Ptr()))
		out.MutateAndWrite(ctx, dID, cb.Custom(func(ctx context.Context, s *api.GlobalState, b *builder.Builder) error {
			b.ReserveMemory(tmp.Range())
			b.Post(value.ObservedPointer(tmp.Address()), buflen, func(r binary.Reader, err error) {
				if err != nil {
					t.onIssue(cmd, id, service.Severity_FatalLevel, fmt.Errorf("Failed to decode glGetShaderSource postback: %v", err))
					return
				}
				r.Data(source)
				if r.Error() != nil {
					t.onIssue(cmd, id, service.Severity_FatalLevel, fmt.Errorf("Failed to decode glGetShaderSource postback: %v", r.Error()))
				}
			})
			return nil
		}))

		out.MutateAndWrite(ctx, dID, cb.GlGetShaderiv(cmd.Shader(), GLenum_GL_COMPILE_STATUS, tmp.Ptr()))
		out.MutateAndWrite(ctx, dID, cb.Custom(func(ctx context.Context, s *api.GlobalState, b *builder.Builder) error {
			b.ReserveMemory(tmp.Range())
			b.Post(value.ObservedPointer(tmp.Address()), 4, func(r binary.Reader, err error) {
				if err != nil {
					t.onIssue(cmd, id, service.Severity_FatalLevel, fmt.Errorf("Failed to decode glGetShaderiv postback: %v", err))
					return
				}
				if r.Uint32() != uint32(GLboolean_GL_TRUE) {
					originalSource := "<unknown>"
					if shader := c.Objects().Shaders().Get(cmd.Shader()); !shader.IsNil() {
						originalSource = shader.Source()
					}
					t.onIssue(cmd, id, service.Severity_ErrorLevel, fmt.Errorf("Shader %d failed to compile. Error:\n%v\nOriginal source:\n%s\nTranslated source:\n%s",
						cmd.Shader(), ntbs(infoLog), text.LineNumber(originalSource), text.LineNumber(ntbs(source))))
				}
				if r.Error() != nil {
					t.onIssue(cmd, id, service.Severity_FatalLevel, fmt.Errorf("Failed to decode glGetShaderiv postback: %v", r.Error()))
				}
			})
			return nil
		}))
		tmp.Free()

	case *GlLinkProgram:
		const buflen = 2048
		tmp := t.state.AllocOrPanic(ctx, 4+buflen)
		out.MutateAndWrite(ctx, dID, cb.GlGetProgramiv(cmd.Program(), GLenum_GL_LINK_STATUS, tmp.Ptr()))
		out.MutateAndWrite(ctx, dID, cb.GlGetProgramInfoLog(cmd.Program(), buflen, memory.Nullptr, tmp.Offset(4)))
		out.MutateAndWrite(ctx, dID, cb.Custom(func(ctx context.Context, s *api.GlobalState, b *builder.Builder) error {
			b.ReserveMemory(tmp.Range())
			b.Post(value.ObservedPointer(tmp.Address()), 4+buflen, func(r binary.Reader, err error) {
				if err != nil {
					t.onIssue(cmd, id, service.Severity_FatalLevel, fmt.Errorf("Failed to decode glGetProgramiv+glGetProgrameInfoLog postback: %v", err))
					return
				}
				msg := make([]byte, buflen)
				res := r.Uint32()
				r.Data(msg)
				if res != uint32(GLboolean_GL_TRUE) {
					vss, fss := "<unknown>", "<unknown>"
					if program := c.Objects().Programs().Get(cmd.Program()); !program.IsNil() {
						if shader := program.Shaders().Get(GLenum_GL_VERTEX_SHADER); !shader.IsNil() {
							vss = shader.Source()
						}
						if shader := program.Shaders().Get(GLenum_GL_FRAGMENT_SHADER); !shader.IsNil() {
							fss = shader.Source()
						}
					}
					logLevel := service.Severity_ErrorLevel
					if pi := FindLinkProgramExtra(s.Arena, cmd.Extras()); !pi.IsNil() && pi.LinkStatus() == GLboolean_GL_TRUE {
						// Increase severity if the program linked successfully during trace.
						logLevel = service.Severity_FatalLevel
					}
					t.onIssue(cmd, id, logLevel, fmt.Errorf("Program %d failed to link. Error:\n%v\n"+
						"Vertex shader source:\n%sFragment shader source:\n%s", cmd.Program(), ntbs(msg),
						text.LineNumber(vss), text.LineNumber(fss)))
				}
			})
			return nil
		}))
		tmp.Free()

	case *GlProgramBinary, *GlProgramBinaryOES, *GlShaderBinary:
		glDev := t.device.Configuration.Drivers.Opengl
		if !canUsePrecompiledShader(c, glDev) {
			t.onIssue(cmd, id, service.Severity_WarningLevel, fmt.Errorf("Pre-compiled binaries cannot be used across on different devices. Capture: %s-%s, Replay: %s-%s",
				c.Constants().Vendor(), c.Constants().Version(), glDev.Vendor, glDev.Version))
		}
	}
}

func (t *findIssues) Flush(ctx context.Context, out transform.Writer) {
	cb := CommandBuilder{Thread: 0, Arena: t.state.Arena}
	out.MutateAndWrite(ctx, api.CmdNoID, cb.Custom(func(ctx context.Context, s *api.GlobalState, b *builder.Builder) error {
		// Since the PostBack function is called before the replay target has actually arrived at the post command,
		// we need to actually write some data here. r.Uint32() is what actually waits for the replay target to have
		// posted the data in question. If we did not do this, we would shut-down the replay as soon as the second-to-last
		// Post had occurred, which may not be anywhere near the end of the stream.
		code := uint32(0xbeefcace)
		b.Push(value.U32(code))
		b.Post(b.Buffer(1), 4, func(r binary.Reader, err error) {
			if err != nil {
				t.res = nil
				log.E(ctx, "Flush did not get expected EOS code: '%v'", err)
				return
			}
			if r.Uint32() != code {
				log.E(ctx, "Flush did not get expected EOS code")
				return
			}
			for _, res := range t.res {
				res(t.issues, nil)
			}
		})
		return nil
	}))
}
<commit_msg>gapis/api/gles: Don't hang if the EOS postback isn't as expected<commit_after>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gles

import (
	"context"
	"fmt"
	"strings"

	"github.com/google/gapid/core/data/binary"
	"github.com/google/gapid/core/log"
	"github.com/google/gapid/core/os/device"
	"github.com/google/gapid/core/text"
	"github.com/google/gapid/gapis/api"
	"github.com/google/gapid/gapis/api/transform"
	"github.com/google/gapid/gapis/capture"
	"github.com/google/gapid/gapis/memory"
	"github.com/google/gapid/gapis/replay"
	"github.com/google/gapid/gapis/replay/builder"
	"github.com/google/gapid/gapis/replay/value"
	"github.com/google/gapid/gapis/service"
	"github.com/google/gapid/gapis/shadertools"
)

// findIssues is a command transform that detects issues when replaying the
// stream of commands. Any issues that are found are written to all the chans in
// the slice out. Once the last issue is sent (if any) all the chans in out are
// closed.
type findIssues struct {
	state         *api.GlobalState
	device        *device.Instance
	targetVersion *Version
	issues        []replay.Issue
	res           []replay.Result
	lastGlError   GLenum
}

func newFindIssues(ctx context.Context, c *capture.Capture, device *device.Instance) *findIssues {
	targetVersion, _ := ParseVersion(device.Configuration.Drivers.Opengl.Version)
	transform := &findIssues{
		state:         c.NewState(ctx),
		device:        device,
		targetVersion: targetVersion,
	}
	transform.state.OnError = func(err interface{}) {
		if glenum, ok := err.(GLenum); ok {
			transform.lastGlError = glenum
		}
	}
	return transform
}

// reportTo adds the chan c to the list of issue listeners.
func (t *findIssues) reportTo(r replay.Result) { t.res = append(t.res, r) }

func (t *findIssues) onIssue(cmd api.Cmd, id api.CmdID, s service.Severity, e error) {
	if s == service.Severity_FatalLevel && isIssueWhitelisted(cmd, e) {
		s = service.Severity_ErrorLevel
	}
	t.issues = append(t.issues, replay.Issue{Command: id, Severity: s, Error: e})
}

// The value 0 is used for many enums - prefer GL_NO_ERROR in this case.
func (e GLenum) ErrorString() string {
	if e == GLenum_GL_NO_ERROR {
		return "GL_NO_ERROR"
	}
	return e.String()
}

type ErrUnexpectedDriverTraceError struct {
	DriverError   GLenum
	ExpectedError GLenum
}

func (e ErrUnexpectedDriverTraceError) Error() string {
	return fmt.Sprintf("%s in trace driver, but we expected %s",
		e.DriverError.ErrorString(), e.ExpectedError.ErrorString())
}

func (t *findIssues) Transform(ctx context.Context, id api.CmdID, cmd api.Cmd, out transform.Writer) {
	ctx = log.Enter(ctx, "findIssues")
	cb := CommandBuilder{Thread: cmd.Thread(), Arena: t.state.Arena}
	t.lastGlError = GLenum_GL_NO_ERROR
	mutateErr := cmd.Mutate(ctx, id, t.state, nil /* no builder */, nil /* no watcher */)

	mutatorsGlError := t.lastGlError
	if e := FindErrorState(cmd.Extras()); e != nil {
		// Check that our API file agrees with the driver which we used for tracking.
		if (e.TraceDriversGlError != GLenum_GL_NO_ERROR) != (mutatorsGlError != GLenum_GL_NO_ERROR) {
			errorMsg := ErrUnexpectedDriverTraceError{
				DriverError:   e.TraceDriversGlError,
				ExpectedError: mutatorsGlError,
			}
			t.onIssue(cmd, id, service.Severity_FatalLevel, errorMsg)
		}
		// Check that the C++ and Go versions of the generated code precisely agree.
		if e.InterceptorsGlError != mutatorsGlError {
			t.onIssue(cmd, id, service.Severity_FatalLevel, fmt.Errorf("%s in interceptor, but we expected %s",
				e.InterceptorsGlError.ErrorString(), mutatorsGlError.ErrorString()))
		}
	}

	if mutateErr != nil {
		// Ignore since downstream transform layers can only consume valid commands
		return
	}

	out.MutateAndWrite(ctx, id, cmd)

	dID := id.Derived()
	s := GetState(t.state)
	c := s.GetContext(cmd.Thread())
	if c.IsNil() {
		return
	}

	// Check the result of glGetError after every command.
	out.MutateAndWrite(ctx, dID, cb.Custom(func(ctx context.Context, s *api.GlobalState, b *builder.Builder) error {
		ptr := b.AllocateTemporaryMemory(4)
		b.Call(funcInfoGlGetError)
		b.Store(ptr)
		b.Post(ptr, 4, builder.Postback(func(r binary.Reader, err error) {
			if err != nil {
				t.onIssue(cmd, id, service.Severity_FatalLevel, fmt.Errorf("Failed to decode glGetError postback: %v", err))
				return
			}
			v := GLenum(r.Uint32())
			err = r.Error()
			if err != nil {
				t.onIssue(cmd, id, service.Severity_FatalLevel, fmt.Errorf("Failed to decode glGetError postback: %v", err))
				return
			}
			if v != GLenum_GL_NO_ERROR {
				t.onIssue(cmd, id, service.Severity_FatalLevel, fmt.Errorf("%v in replay driver", v))
			}
		}))
		return nil
	}))

	// null-terminated byte slice to string
	ntbs := func(b []byte) string {
		s := string(b)
		for i, r := range s {
			if r == 0 {
				return strings.TrimSpace(s[:i])
			}
		}
		return strings.TrimSpace(s)
	}

	switch cmd := cmd.(type) {
	case *GlCompileShader:
		shader := c.Objects().Shaders().Get(cmd.Shader())
		st, err := shader.Type().ShaderType()
		if err != nil {
			t.onIssue(cmd, id, service.Severity_ErrorLevel, err)
			return
		}

		if t.targetVersion.IsES {
			// Check we are able to convert this GLES shader to desktop GL.
			opts := shadertools.ConvertOptions{
				ShaderType:        st,
				CheckAfterChanges: true,
				Disassemble:       true,
				TargetGLSLVersion: 430,
			}

			if _, err := shadertools.ConvertGlsl(shader.Source(), &opts); err != nil {
				t.onIssue(cmd, id, service.Severity_ErrorLevel, err)
			}
		}

		const buflen = 8192
		tmp := t.state.AllocOrPanic(ctx, buflen)

		infoLog := make([]byte, buflen)
		out.MutateAndWrite(ctx, dID, cb.GlGetShaderInfoLog(cmd.Shader(), buflen, memory.Nullptr, tmp.Ptr()))
		out.MutateAndWrite(ctx, dID, cb.Custom(func(ctx context.Context, s *api.GlobalState, b *builder.Builder) error {
			b.ReserveMemory(tmp.Range())
			b.Post(value.ObservedPointer(tmp.Address()), buflen, func(r binary.Reader, err error) {
				if err != nil {
					t.onIssue(cmd, id, service.Severity_FatalLevel, fmt.Errorf("Failed to decode glGetShaderInfoLog postback: %v", err))
					return
				}
				r.Data(infoLog)
				if r.Error() != nil {
					t.onIssue(cmd, id, service.Severity_FatalLevel, fmt.Errorf("Failed to decode glGetShaderInfoLog postback: %v", r.Error()))
				}
			})
			return nil
		}))

		source := make([]byte, buflen)
		out.MutateAndWrite(ctx, dID, cb.GlGetShaderSource(cmd.Shader(), buflen, memory.Nullptr, tmp.Ptr()))
		out.MutateAndWrite(ctx, dID, cb.Custom(func(ctx context.Context, s *api.GlobalState, b *builder.Builder) error {
			b.ReserveMemory(tmp.Range())
			b.Post(value.ObservedPointer(tmp.Address()), buflen, func(r binary.Reader, err error) {
				if err != nil {
					t.onIssue(cmd, id, service.Severity_FatalLevel, fmt.Errorf("Failed to decode glGetShaderSource postback: %v", err))
					return
				}
				r.Data(source)
				if r.Error() != nil {
					t.onIssue(cmd, id, service.Severity_FatalLevel, fmt.Errorf("Failed to decode glGetShaderSource postback: %v", r.Error()))
				}
			})
			return nil
		}))

		out.MutateAndWrite(ctx, dID, cb.GlGetShaderiv(cmd.Shader(), GLenum_GL_COMPILE_STATUS, tmp.Ptr()))
		out.MutateAndWrite(ctx, dID, cb.Custom(func(ctx context.Context, s *api.GlobalState, b *builder.Builder) error {
			b.ReserveMemory(tmp.Range())
			b.Post(value.ObservedPointer(tmp.Address()), 4, func(r binary.Reader, err error) {
				if err != nil {
					t.onIssue(cmd, id, service.Severity_FatalLevel, fmt.Errorf("Failed to decode glGetShaderiv postback: %v", err))
					return
				}
				if r.Uint32() != uint32(GLboolean_GL_TRUE) {
					originalSource := "<unknown>"
					if shader := c.Objects().Shaders().Get(cmd.Shader()); !shader.IsNil() {
						originalSource = shader.Source()
					}
					t.onIssue(cmd, id, service.Severity_ErrorLevel, fmt.Errorf("Shader %d failed to compile. Error:\n%v\nOriginal source:\n%s\nTranslated source:\n%s",
						cmd.Shader(), ntbs(infoLog), text.LineNumber(originalSource), text.LineNumber(ntbs(source))))
				}
				if r.Error() != nil {
					t.onIssue(cmd, id, service.Severity_FatalLevel, fmt.Errorf("Failed to decode glGetShaderiv postback: %v", r.Error()))
				}
			})
			return nil
		}))
		tmp.Free()

	case *GlLinkProgram:
		const buflen = 2048
		tmp := t.state.AllocOrPanic(ctx, 4+buflen)
		out.MutateAndWrite(ctx, dID, cb.GlGetProgramiv(cmd.Program(), GLenum_GL_LINK_STATUS, tmp.Ptr()))
		out.MutateAndWrite(ctx, dID, cb.GlGetProgramInfoLog(cmd.Program(), buflen, memory.Nullptr, tmp.Offset(4)))
		out.MutateAndWrite(ctx, dID, cb.Custom(func(ctx context.Context, s *api.GlobalState, b *builder.Builder) error {
			b.ReserveMemory(tmp.Range())
			b.Post(value.ObservedPointer(tmp.Address()), 4+buflen, func(r binary.Reader, err error) {
				if err != nil {
					t.onIssue(cmd, id, service.Severity_FatalLevel, fmt.Errorf("Failed to decode glGetProgramiv+glGetProgrameInfoLog postback: %v", err))
					return
				}
				msg := make([]byte, buflen)
				res := r.Uint32()
				r.Data(msg)
				if res != uint32(GLboolean_GL_TRUE) {
					vss, fss := "<unknown>", "<unknown>"
					if program := c.Objects().Programs().Get(cmd.Program()); !program.IsNil() {
						if shader := program.Shaders().Get(GLenum_GL_VERTEX_SHADER); !shader.IsNil() {
							vss = shader.Source()
						}
						if shader := program.Shaders().Get(GLenum_GL_FRAGMENT_SHADER); !shader.IsNil() {
							fss = shader.Source()
						}
					}
					logLevel := service.Severity_ErrorLevel
					if pi := FindLinkProgramExtra(s.Arena, cmd.Extras()); !pi.IsNil() && pi.LinkStatus() == GLboolean_GL_TRUE {
						// Increase severity if the program linked successfully during trace.
						logLevel = service.Severity_FatalLevel
					}
					t.onIssue(cmd, id, logLevel, fmt.Errorf("Program %d failed to link. Error:\n%v\n"+
						"Vertex shader source:\n%sFragment shader source:\n%s", cmd.Program(), ntbs(msg),
						text.LineNumber(vss), text.LineNumber(fss)))
				}
			})
			return nil
		}))
		tmp.Free()

	case *GlProgramBinary, *GlProgramBinaryOES, *GlShaderBinary:
		glDev := t.device.Configuration.Drivers.Opengl
		if !canUsePrecompiledShader(c, glDev) {
			t.onIssue(cmd, id, service.Severity_WarningLevel, fmt.Errorf("Pre-compiled binaries cannot be used across on different devices. Capture: %s-%s, Replay: %s-%s",
				c.Constants().Vendor(), c.Constants().Version(), glDev.Vendor, glDev.Version))
		}
	}
}

func (t *findIssues) Flush(ctx context.Context, out transform.Writer) {
	cb := CommandBuilder{Thread: 0, Arena: t.state.Arena}
	out.MutateAndWrite(ctx, api.CmdNoID, cb.Custom(func(ctx context.Context, s *api.GlobalState, b *builder.Builder) error {
		// Since the PostBack function is called before the replay target has actually arrived at the post command,
		// we need to actually write some data here. r.Uint32() is what actually waits for the replay target to have
		// posted the data in question. If we did not do this, we would shut-down the replay as soon as the second-to-last
		// Post had occurred, which may not be anywhere near the end of the stream.
		code := uint32(0xbeefcace)
		b.Push(value.U32(code))
		b.Post(b.Buffer(1), 4, func(r binary.Reader, err error) {
			for _, res := range t.res {
				res.Do(func() (interface{}, error) {
					if err != nil {
						return nil, log.Err(ctx, err, "Flush did not get expected EOS code: '%v'")
					}
					if r.Uint32() != code {
						return nil, log.Err(ctx, nil, "Flush did not get expected EOS code")
					}
					return t.issues, nil
				})
			}
		})
		return nil
	}))
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"github.com/skeswa/gophr/common"
	"io/ioutil"
	"log"
	"strings"
	"time"
)

type GoPackage struct {
	Description    string
	GitHubURL      string
	GoDocURL       string
	IndexTime      string
	HttpStatusCode int
	AwesomeGo      bool
	Versions       []string
}

const (
	refsFetchURLTemplate = "https://%s.git/info/refs?service=git-upload-pack"
)

func main() {
	log.Println("Started Download of Godoc/index")

	doc, err := goquery.NewDocument("https://godoc.org/-/index")
	if err != nil {
		log.Fatal(err)
	}

	log.Println("Finished Download of Godoc/index")
	log.Println("Started scraping all github packages from Godoc/index")

	var goPackageList = make([]*GoPackage, 0)
	var goPackageMap = make(map[string]*GoPackage)

	// For each tr element on the page
	doc.Find("tr").Each(func(i int, s *goquery.Selection) {
		children := s.Children()
		goPackage := new(GoPackage)

		// For each child in the tr element
		children.Each(func(i int, s2 *goquery.Selection) {
			childURL, childURLexists := s.Find("a").Attr("href")
			childDescription := s.Text()

			if childURLexists == true {
				goPackage.GitHubURL = strings.Trim(childURL, "/")
			}

			if len(childDescription) > 0 {
				// TODO check if description isn't just the url, if so dont set it
				goPackage.Description = childDescription
			}
		})

		// Only continue if this goPackage contains a GitHub URL
		// TODO check to make sure github.com is the prefix
		if strings.Contains(goPackage.GitHubURL, "github.com") {
			// Build go doc url
			goPackage.GoDocURL = ("https://godoc.org/" + goPackage.GitHubURL)

			// Create Index Time
			t := time.Now()
			time := t.String()
			goPackage.IndexTime = time

			// Add goPackage to goPackageList
			goPackageList = append(goPackageList, goPackage)
			// Hash the goPackage.GitHubURL for lookup
			goPackageMap[goPackage.GitHubURL] = goPackage
		}
	})

	log.Println("Finished scraping all github packages from Godoc/index")
	log.Println("Started Download of awesome-go/README.md")

	doc, err = goquery.NewDocument("https://godoc.org/-/index")
	if err != nil {
		log.Fatal(err)
	}

	log.Println("Finished Download of awesome-go/README.md")
	log.Println("Started scraping awesome-go/README.md")

	doc.Find("a").Each(func(i int, s *goquery.Selection) {
		childURL, _ := s.Attr("href")

		goPackage, goPackageExists := goPackageMap[childURL]

		if goPackageExists {
			goPackage.AwesomeGo = true
			goPackageMap[childURL] = goPackage
			printGoPackage(goPackage)
		}
	})

	log.Println("Finished scraping awesome-go/README.md")

	for _, goPackage := range goPackageMap {
		refs, err := common.FetchRefs(goPackage.GitHubURL)
		if err != nil {
			log.Println("ERROR", goPackage.GitHubURL, " failed to return.", err)
		}

		var versions []string
		for _, version := range refs.Candidates {
			versions = append(versions, version.String())
		}

		goPackage.Versions = versions
	}

	log.Println("Finished Building GoPackages")
	log.Println("Creating JSON dump")
	createJSONDump(goPackageList)
}

// Create a tmp JSON dump of all serialized goPackageData
func createJSONDump(goPackageList []*GoPackage) {
	var buffer bytes.Buffer

	for _, goPackage := range goPackageList {
		buffer.WriteString("{\"url\": \"" + goPackage.GitHubURL + "\", \"description\": \"" + goPackage.Description + "\", \"index_time\": \"" + goPackage.IndexTime + "\" },\n")
	}

	t := time.Now()
	time := t.String()
	jsonData := []byte(buffer.String())
	_ = ioutil.WriteFile("./"+time+"tmp.json", jsonData, 0644)
}

func printGoPackage(gp *GoPackage) {
	fmt.Println("GitHub URL = ", gp.GitHubURL)
	fmt.Println("Description = ", gp.Description)
	fmt.Println("Index time = ", gp.IndexTime)
	fmt.Println("GoDocURL = ", gp.GoDocURL)
	fmt.Println("AwesomeGo = ", gp.AwesomeGo, "\n")
}
<commit_msg>use go routine to make requests to github, plus general clean up of indexer<commit_after>package main

import (
	"bytes"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"github.com/skeswa/gophr/common"
	"io/ioutil"
	"log"
	"strings"
	"time"
)

type GoPackage struct {
	Description    string
	GitHubURL      string
	GoDocURL       string
	IndexTime      string
	HttpStatusCode int
	AwesomeGo      bool
	Versions       []string
}

func main() {
	log.Println("Started Download of Godoc/index")

	doc, err := goquery.NewDocument("https://godoc.org/-/index")
	if err != nil {
		log.Fatal(err)
	}

	log.Println("Finished Download of Godoc/index\n")
	log.Println("Started scraping all github packages from Godoc/index")

	var goPackageMap = make(map[string]*GoPackage)

	// For each tr element on the page
	doc.Find("tr").Each(func(i int, s *goquery.Selection) {
		children := s.Children()
		goPackage := new(GoPackage)

		// For each child in the tr element
		children.Each(func(i int, s2 *goquery.Selection) {
			childURL, childURLexists := s.Find("a").Attr("href")
			childDescription := s.Text()

			if childURLexists == true {
				goPackage.GitHubURL = strings.Trim(childURL, "/")
			}

			if len(childDescription) > 0 {
				// TODO check if description isn't just the url, if so dont set it
				goPackage.Description = childDescription
			}
		})

		// Only continue if this goPackage contains a GitHub URL
		// TODO check to make sure github.com is the prefix
		gitHubUrlTokens := strings.Split(goPackage.GitHubURL, "/")

		if strings.Contains(goPackage.GitHubURL, "github.com") && len(gitHubUrlTokens) == 3 {
			// Build go doc url
			goPackage.GoDocURL = ("https://godoc.org/" + goPackage.GitHubURL)

			// Create Index Time
			t := time.Now()
			time := t.String()
			goPackage.IndexTime = time

			goPackageMap[goPackage.GitHubURL] = goPackage
		}
	})

	log.Println("Finished scraping ", len(goPackageMap), " github packages from Godoc/index\n")
	log.Println("Started Download of awesome-go/README.md")

	doc, err = goquery.NewDocument("https://godoc.org/-/index")
	if err != nil {
		log.Fatal(err)
	}

	log.Println("Finished Download of awesome-go/README.md\n")
	log.Println("Started scraping awesome-go/README.md")

	awesomeGoCount := 0
	doc.Find("a").Each(func(i int, s *goquery.Selection) {
		childURL, _ := s.Attr("href")
		fmt.Println(childURL)
		goPackage, goPackageExists := goPackageMap[childURL]

		if goPackageExists {
			goPackage.AwesomeGo = true
			goPackageMap[childURL] = goPackage
			awesomeGoCount++
		}
	})

	log.Println("Finished scraping ", awesomeGoCount, " awesome-go/README.md\n")
	log.Println("Started retrieving versions for go packages")

	resc, errc := make(chan *GoPackage), make(chan *GoPackage)
	var goErrorPackageMap = make(map[string]*GoPackage)

	for _, goPackage := range goPackageMap {
		go func(goPackage *GoPackage) {
			refs, err := common.FetchRefs(goPackage.GitHubURL)
			if err != nil {
				log.Println("ERROR", goPackage.GitHubURL, " failed to return.", err)
				errc <- goPackage
				return
			}

			var versions []string
			for _, version := range refs.Candidates {
				versions = append(versions, version.String())
			}
			goPackage.Versions = versions
			resc <- goPackage
		}(goPackage)
	}

	// Create Errored out Map
	length := len(goPackageMap)
	for i := 0; i < length; i++ {
		select {
		case goPackage := <-resc:
			goPackageMap[goPackage.GitHubURL] = goPackage
		case errPackage := <-errc:
			delete(goPackageMap, errPackage.GitHubURL)
			goErrorPackageMap[errPackage.GitHubURL] = errPackage
		}
	}

	log.Println("Finished Building ", len(goPackageMap), " GoPackages")
	log.Println("Creating JSON dump of ", len(goPackageMap), " valid go packages")
	createJSONDump(goPackageMap, "valid-go-packages")
	log.Println("Finished creating JSON dump\n")

	log.Println(len(goErrorPackageMap), " GoPackages where not found on github")
	log.Println("Creating JSON dump of ", len(goPackageMap), " err packages")
	createJSONDump(goErrorPackageMap, "invalid-go-packages")
	log.Println("Finished creating JSON dump")
}

// Create a tmp JSON dump of all serialized goPackageData
func createJSONDump(goPackageMap map[string]*GoPackage, fileName string) {
	var buffer bytes.Buffer

	for _, goPackage := range goPackageMap {
		buffer.WriteString("{\"url\": \"" + goPackage.GitHubURL + "\", \"description\": \"" + goPackage.Description + "\", \"index_time\": \"" + goPackage.IndexTime + "\" }, \"versions\": \"" + fmt.Sprintf("%v", goPackage.Versions) + "\" \n")
	}

	t := time.Now()
	time := strings.Replace(t.String(), " ", "", -1)
	jsonData := []byte(buffer.String())
	_ = ioutil.WriteFile("./"+time+"-"+fileName+".json", jsonData, 0644)
}

func printGoPackage(goPackage *GoPackage) {
	fmt.Printf("%v", goPackage)
}
<|endoftext|>


<commit_before>//+build integration

package integration

import (
	"encoding/pem"
	"flag"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"testing"
	"time"

	ct "github.com/google/certificate-transparency/go"
	"github.com/google/certificate-transparency/go/client"
	"github.com/google/certificate-transparency/go/jsonclient"
	"golang.org/x/net/context"
)

var httpServerFlag = flag.String("ct_http_server", "localhost:8092", "Server address:port")
var pubKey = flag.String("public_key_file", "", "Name of file containing log's public key")
var testdata = flag.String("testdata", "testdata", "Name of directory with test data")

func TestCTIntegration(t *testing.T) {
	flag.Parse()

	logURI := "http://" + (*httpServerFlag)

	opts := jsonclient.Options{}
	if *pubKey != "" {
		pubkey, err := ioutil.ReadFile(*pubKey)
		if err != nil {
			t.Fatalf("Failed to get public key contents: %v", err)
		}
		opts.PublicKey = string(pubkey)
	}
	logClient, err := client.New(logURI, nil, opts)
	if err != nil {
		t.Fatalf("Failed to create LogClient instance: %v", err)
	}
	ctx := context.Background()

	// Stage 0: get accepted roots, which should just be the fake CA.
	roots, err := logClient.GetAcceptedRoots(ctx)
	if err != nil {
		t.Fatalf("Failed to get roots: %v", err)
	}
	if len(roots) != 1 {
		t.Errorf("len(GetAcceptableRoots())=%d; want 1", len(roots))
	}

	// Stage 1: get the STH, which should be empty.
	sth, err := logClient.GetSTH(ctx)
	if err != nil {
		t.Fatalf("Failed to get STH: %v", err)
	}
	if sth.Version != 0 {
		t.Errorf("sth.Version=%v; want V1(0)", sth.Version)
	}
	if sth.TreeSize != 0 {
		t.Errorf("sth.TreeSize=%d; want 0", sth.TreeSize)
	}
	when := ctTimestampToTime(sth.Timestamp)
	fmt.Printf("%v: Got STH(size=%d): roothash=%x\n", when, sth.TreeSize, sth.SHA256RootHash)
	fmt.Printf("%v\n", signatureToString(&sth.TreeHeadSignature))

	// Stage 2: add a single cert (the intermediate CA), get an SCT.
	certdata, err := ioutil.ReadFile(filepath.Join(*testdata, "int-ca.cert"))
	if err != nil {
		t.Fatalf("Failed to load certificate: %v", err.Error())
	}
	sct, err := logClient.AddChain(ctx, certsFromPEM(certdata))
	if err != nil {
		t.Fatalf("Failed to AddChain(0): %v", err)
	}
	// Display the SCT
	when = ctTimestampToTime(sct.Timestamp)
	fmt.Printf("%v: Uploaded certs to %v log, got SCT:\n", when, sct.SCTVersion)
	fmt.Printf("%v\n", signatureToString(&sct.Signature))

	// Stage 3: keep getting the STH until tree size becomes 1.
	// TODO(drysdale)

	// Stage 4: get a consistency proof from 0->1.
	// TODO(drysdale)

	// Stage 5: add certificates 2, 3, 4, 5,...N, for some random N in [4,25]
	// TODO(drysdale)

	// Stage 6: keep getting the STH until tree size becomes N.
	// TODO(drysdale)

	// Stage 7: get a consistency proof from 1->N.
	// TODO(drysdale)

	// Stage 8: get entries [1, N]
	// TODO(drysdale)

	// Stage 9: get an audit proof for cert M, randomly chosen in [1,N]
	// TODO(drysdale)
}

func ctTimestampToTime(ts uint64) time.Time {
	secs := int64(ts / 1000)
	msecs := int64(ts % 1000)
	return time.Unix(secs, msecs*1000000)
}

func signatureToString(signed *ct.DigitallySigned) string {
	return fmt.Sprintf("Signature: Hash=%v Sign=%v Value=%x", signed.Algorithm.Hash, signed.Algorithm.Signature, signed.Signature)
}

func certsFromPEM(data []byte) []ct.ASN1Cert {
	var chain []ct.ASN1Cert
	for {
		var block *pem.Block
		block, data = pem.Decode(data)
		if block == nil {
			break
		}
		if block.Type == "CERTIFICATE" {
			chain = append(chain, ct.ASN1Cert{Data: block.Bytes})
		}
	}
	return chain
}
<commit_msg>integration/ct: get-sth-consistency<commit_after>//+build integration

package integration

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/pem"
	"flag"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"testing"
	"time"

	ct "github.com/google/certificate-transparency/go"
	"github.com/google/certificate-transparency/go/client"
	"github.com/google/certificate-transparency/go/jsonclient"
	"golang.org/x/net/context"
)

var httpServerFlag = flag.String("ct_http_server", "localhost:8092", "Server address:port")
var pubKey = flag.String("public_key_file", "", "Name of file containing log's public key")
var testdata = flag.String("testdata", "testdata", "Name of directory with test data")

func TestCTIntegration(t *testing.T) {
	flag.Parse()

	logURI := "http://" + (*httpServerFlag)

	opts := jsonclient.Options{}
	if *pubKey != "" {
		pubkey, err := ioutil.ReadFile(*pubKey)
		if err != nil {
			t.Fatalf("Failed to get public key contents: %v", err)
		}
		opts.PublicKey = string(pubkey)
	}
	logClient, err := client.New(logURI, nil, opts)
	if err != nil {
		t.Fatalf("Failed to create LogClient instance: %v", err)
	}
	ctx := context.Background()

	// Stage 0: get accepted roots, which should just be the fake CA.
	roots, err := logClient.GetAcceptedRoots(ctx)
	if err != nil {
		t.Fatalf("Failed to get roots: %v", err)
	}
	if len(roots) != 1 {
		t.Errorf("len(GetAcceptableRoots())=%d; want 1", len(roots))
	}

	// Stage 1: get the STH, which should be empty.
	sth, err := logClient.GetSTH(ctx)
	if err != nil {
		t.Fatalf("Failed to get STH: %v", err)
	}
	if sth.Version != 0 {
		t.Errorf("sth.Version=%v; want V1(0)", sth.Version)
	}
	if sth.TreeSize != 0 {
		t.Errorf("sth.TreeSize=%d; want 0", sth.TreeSize)
	}
	fmt.Printf("%v: Got STH(size=%d): roothash=%x\n", ctTime(sth.Timestamp), sth.TreeSize, sth.SHA256RootHash)

	// Stage 2: add a single cert (the intermediate CA), get an SCT.
	chain, err := getChain("int-ca.cert")
	if err != nil {
		t.Fatalf("Failed to load certificate: %v", err)
	}
	sct, err := logClient.AddChain(ctx, chain)
	if err != nil {
		t.Fatalf("Failed to AddChain(): %v", err)
	}
	// Display the SCT
	fmt.Printf("%v: Uploaded cert to %v log, got SCT\n", ctTime(sct.Timestamp), sct.SCTVersion)

	// Keep getting the STH until tree size becomes 1.
	sth1, err := awaitTreeSize(ctx, logClient, 1, true)
	if err != nil {
		t.Fatalf("Failed to get STH for size=1: %v", err)
	}

	// Stage 3: add a second cert, wait for tree size = 2
	chain, err = getChain("leaf01.chain")
	if err != nil {
		t.Fatalf("Failed to load certificate: %v", err)
	}
	sct, err = logClient.AddChain(ctx, chain)
	if err != nil {
		t.Fatalf("Failed to AddChain(): %v", err)
	}
	fmt.Printf("%v: Uploaded cert to %v log, got SCT\n", ctTime(sct.Timestamp), sct.SCTVersion)
	sth2, err := awaitTreeSize(ctx, logClient, 2, true)
	if err != nil {
		t.Fatalf("Failed to get STH for size=1: %v", err)
	}

	// Stage 4: get a consistency proof from size 1-> size 2.
	proof, err := logClient.GetSTHConsistency(ctx, 1, 2)
	if err != nil {
		t.Fatalf("Failed to GetSTHConsistency(): %v", err)
	}
	//                 sth2
	//                 / \
	//  sth1   =>      a b
	//    |            | |
	//   d0           d0 d1
	// So consistency proof is [b] and we should have:
	//   sth2 == SHA256(0x01 | sth1 | b)
	if len(proof) != 1 {
		t.Fatalf("len(proof)=%d; want 1", len(proof))
	}
	if len(proof[0]) != sha256.Size {
		t.Fatalf("len(proof[0])=%d; want %d", len(proof[0]), sha256.Size)
	}
	fmt.Printf("Proof 1->2 = %x\n", proof)
	data := make([]byte, 1+sha256.Size+sha256.Size)
	data[0] = 0x01
	copy(data[1:1+sha256.Size], sth1.SHA256RootHash[:])
	copy(data[1+sha256.Size:], proof[0])
	if want, got := sha256.Sum256(data), sth2.SHA256RootHash; got != want {
		t.Errorf("SHA256(1|sth1|proof[0])=%s; want %s", hex.EncodeToString(got[:]), hex.EncodeToString(want[:]))
	}

	// Stage 5: add certificates 2, 3, 4, 5,...N, for some random N in [4,25]
	// TODO(drysdale)

	// Stage 6: keep getting the STH until tree size becomes N.
	// TODO(drysdale)

	// Stage 7: get a consistency proof from 1->N.
	// TODO(drysdale)

	// Stage 8: get entries [1, N]
	// TODO(drysdale)

	// Stage 9: get an audit proof for cert M, randomly chosen in [1,N]
	// TODO(drysdale)
}

func ctTime(ts uint64) time.Time {
	secs := int64(ts / 1000)
	msecs := int64(ts % 1000)
	return time.Unix(secs, msecs*1000000)
}

func signatureToString(signed *ct.DigitallySigned) string {
	return fmt.Sprintf("Signature: Hash=%v Sign=%v Value=%x", signed.Algorithm.Hash, signed.Algorithm.Signature, signed.Signature)
}

func certsFromPEM(data []byte) []ct.ASN1Cert {
	var chain []ct.ASN1Cert
	for {
		var block *pem.Block
		block, data = pem.Decode(data)
		if block == nil {
			break
		}
		if block.Type == "CERTIFICATE" {
			chain = append(chain, ct.ASN1Cert{Data: block.Bytes})
		}
	}
	return chain
}

func getChain(path string) ([]ct.ASN1Cert, error) {
	certdata, err := ioutil.ReadFile(filepath.Join(*testdata, path))
	if err != nil {
		return nil, fmt.Errorf("Failed to load certificate: %v", err)
	}
	return certsFromPEM(certdata), nil
}

func awaitTreeSize(ctx context.Context, logClient *client.LogClient, size uint64, exact bool) (*ct.SignedTreeHead, error) {
	var sth *ct.SignedTreeHead
	for sth == nil || sth.TreeSize < size {
		time.Sleep(200 * time.Millisecond)
		var err error
		sth, err = logClient.GetSTH(ctx)
		if err != nil {
			return nil, fmt.Errorf("failed to get STH: %v", err)
		}
	}
	if exact && sth.TreeSize != size {
		return nil, fmt.Errorf("sth.TreeSize=%d; want 1", sth.TreeSize)
	}
	fmt.Printf("%v: Got STH(size=%d): roothash=%x\n", ctTime(sth.Timestamp), sth.TreeSize, sth.SHA256RootHash)
	return sth, nil
}
<|endoftext|>


<commit_before>package main

type Res struct {
	Cmd *ResCmd
	ProcState *ResProcState
}

type ResCmd struct {
	Pid int
}

type ResProcState struct {
	Pid int
	Exited bool
	ExitStatus int
	Signaled bool
	TermSig int
	CoreDump bool
	Stopped bool
	StopSig int
	Continued bool
}

func RecvRes() (r Res, err error) {
	err = ResDecoder.Decode(&r)
	return
}
<commit_msg>das: Accept response of type BadRequest<commit_after>package main

type Res struct {
	Cmd *ResCmd
	ProcState *ResProcState
	BadRequest *ResBadRequest
}

type ResCmd struct {
	Pid int
}

type ResProcState struct {
	Pid int
	Exited bool
	ExitStatus int
	Signaled bool
	TermSig int
	CoreDump bool
	Stopped bool
	StopSig int
	Continued bool
}

type ResBadRequest struct {
	Err string
}

func RecvRes() (r Res, err error) {
	err = ResDecoder.Decode(&r)
	return
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package provisioner

import (
	"fmt"
	"sync/atomic"

	"github.com/juju/utils/fslock"

	"github.com/juju/juju/agent"
	"github.com/juju/juju/container"
	"github.com/juju/juju/container/kvm"
	"github.com/juju/juju/container/lxc"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/instance"
	"github.com/juju/juju/state"
	"github.com/juju/juju/state/api/params"
	apiprovisioner "github.com/juju/juju/state/api/provisioner"
	"github.com/juju/juju/state/api/watcher"
	"github.com/juju/juju/worker"
)

// ContainerSetup is a StringsWatchHandler that is notified when containers
// are created on the given machine. It will set up the machine to be able
// to create containers and start a suitable provisioner.
type ContainerSetup struct {
	runner              worker.Runner
	supportedContainers []instance.ContainerType
	provisioner         *apiprovisioner.State
	machine             *apiprovisioner.Machine
	config              agent.Config
	initLock            *fslock.Lock

	// Save the workerName so the worker thread can be stopped.
	workerName string
	// setupDone[containerType] is non zero if the container setup has been invoked
	// for that container type.
	setupDone map[instance.ContainerType]*int32
	// The number of provisioners started. Once all necessary provisioners have
	// been started, the container watcher can be stopped.
	numberProvisioners int32
}

// NewContainerSetupHandler returns a StringsWatchHandler which is notified when
// containers are created on the given machine.
func NewContainerSetupHandler(runner worker.Runner, workerName string, supportedContainers []instance.ContainerType,
	machine *apiprovisioner.Machine, provisioner *apiprovisioner.State,
	config agent.Config, initLock *fslock.Lock) worker.StringsWatchHandler {

	return &ContainerSetup{
		runner:              runner,
		machine:             machine,
		supportedContainers: supportedContainers,
		provisioner:         provisioner,
		config:              config,
		workerName:          workerName,
		initLock:            initLock,
	}
}

// SetUp is defined on the StringsWatchHandler interface.
func (cs *ContainerSetup) SetUp() (watcher watcher.StringsWatcher, err error) {
	// Set up the semaphores for each container type.
	cs.setupDone = make(map[instance.ContainerType]*int32, len(instance.ContainerTypes))
	for _, containerType := range instance.ContainerTypes {
		zero := int32(0)
		cs.setupDone[containerType] = &zero
	}
	// Listen to all container lifecycle events on our machine.
	if watcher, err = cs.machine.WatchAllContainers(); err != nil {
		return nil, err
	}
	return watcher, nil
}

// Handle is called whenever containers change on the machine being watched.
// Machines start out with no containers so the first time Handle is called,
// it will be because a container has been added.
func (cs *ContainerSetup) Handle(containerIds []string) (resultError error) {
	// Consume the initial watcher event.
	if len(containerIds) == 0 {
		return nil
	}

	logger.Infof("initial container setup with ids: %v", containerIds)
	for _, id := range containerIds {
		containerType := state.ContainerTypeFromId(id)
		// If this container type has been dealt with, do nothing.
		if atomic.LoadInt32(cs.setupDone[containerType]) != 0 {
			continue
		}
		if err := cs.initialiseAndStartProvisioner(containerType); err != nil {
			logger.Errorf("starting container provisioner for %v: %v", containerType, err)
			// Just because dealing with one type of container fails, we won't exit the entire
			// function because we still want to try and start other container types. So we
			// take note of and return the first such error.
			if resultError == nil {
				resultError = err
			}
		}
	}
	return resultError
}

func (cs *ContainerSetup) initialiseAndStartProvisioner(containerType instance.ContainerType) error {
	// Flag that this container type has been handled.
	atomic.StoreInt32(cs.setupDone[containerType], 1)

	if atomic.AddInt32(&cs.numberProvisioners, 1) == int32(len(cs.supportedContainers)) {
		// We only care about the initial container creation.
		// This worker has done its job so stop it.
		// We do not expect there will be an error, and there's not much we can do anyway.
		if err := cs.runner.StopWorker(cs.workerName); err != nil {
			logger.Warningf("stopping machine agent container watcher: %v", err)
		}
	}

	if initialiser, broker, err := cs.getContainerArtifacts(containerType); err != nil {
		return fmt.Errorf("initialising container infrastructure on host machine: %v", err)
	} else {
		if err := cs.runInitialiser(containerType, initialiser); err != nil {
			return fmt.Errorf("setting up container dependencies on host machine: %v", err)
		}
		return StartProvisioner(cs.runner, containerType, cs.provisioner, cs.config, broker)
	}
}

// runInitialiser runs the container initialiser with the initialisation hook held.
func (cs *ContainerSetup) runInitialiser(containerType instance.ContainerType, initialiser container.Initialiser) error {
	if err := cs.initLock.Lock(fmt.Sprintf("initialise-%s", containerType)); err != nil {
		return fmt.Errorf("failed to acquire initialization lock: %v", err)
	}
	defer cs.initLock.Unlock()
	return initialiser.Initialise()
}

// TearDown is defined on the StringsWatchHandler interface.
func (cs *ContainerSetup) TearDown() error {
	// Nothing to do here.
	return nil
}

func (cs *ContainerSetup) getContainerArtifacts(containerType instance.ContainerType) (container.Initialiser, environs.InstanceBroker, error) {
	tools, err := cs.provisioner.Tools(cs.config.Tag())
	if err != nil {
		logger.Errorf("cannot get tools from machine for %s container", containerType)
		return nil, nil, err
	}
	var initialiser container.Initialiser
	var broker environs.InstanceBroker

	managerConfig, err := containerManagerConfig(containerType, cs.provisioner, cs.config)
	if err != nil {
		return nil, nil, err
	}

	switch containerType {
	case instance.LXC:
		series, err := cs.machine.Series()
		if err != nil {
			return nil, nil, err
		}

		initialiser = lxc.NewContainerInitialiser(series)
		broker, err = NewLxcBroker(cs.provisioner, tools, cs.config, managerConfig)
		if err != nil {
			return nil, nil, err
		}
	case instance.KVM:
		initialiser = kvm.NewContainerInitialiser()
		broker, err = NewKvmBroker(cs.provisioner, tools, cs.config, managerConfig)
		if err != nil {
			logger.Errorf("failed to create new kvm broker")
			return nil, nil, err
		}
	default:
		return nil, nil, fmt.Errorf("unknown container type: %v", containerType)
	}
	return initialiser, broker, nil
}

func containerManagerConfig(
	containerType instance.ContainerType,
	provisioner *apiprovisioner.State,
	agentConfig agent.Config,
) (container.ManagerConfig, error) {
	// Ask the provisioner for the container manager configuration.
	managerConfigResult, err := provisioner.ContainerManagerConfig(
		params.ContainerManagerConfigParams{Type: containerType},
	)
	if params.IsCodeNotImplemented(err) {
		// We currently don't support upgrading;
		// revert to the old configuration.
		managerConfigResult.ManagerConfig = container.ManagerConfig{container.ConfigName: "juju"}
	}
	if err != nil {
		return nil, err
	}
	// If a namespace is specified, that should instead be used as the config name.
	if namespace := agentConfig.Value(agent.Namespace); namespace != "" {
		managerConfigResult.ManagerConfig[container.ConfigName] = namespace
	}
	managerConfig := container.ManagerConfig(managerConfigResult.ManagerConfig)
	return managerConfig, nil
}

// Override for testing.
var StartProvisioner = startProvisionerWorker

// startProvisionerWorker kicks off a provisioner task responsible for creating containers
// of the specified type on the machine.
func startProvisionerWorker(runner worker.Runner, containerType instance.ContainerType,
	provisioner *apiprovisioner.State, config agent.Config, broker environs.InstanceBroker) error {

	workerName := fmt.Sprintf("%s-provisioner", containerType)
	// The provisioner task is created after a container record has already been added to the machine.
	// It will see that the container does not have an instance yet and create one.
	return runner.StartWorker(workerName, func() (worker.Worker, error) {
		return NewContainerProvisioner(containerType, provisioner, config, broker), nil
	})
}
<commit_msg>Add extra debugging to container provisioner and tweak startup<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package provisioner

import (
	"fmt"
	"sync/atomic"

	"github.com/juju/utils/fslock"
	"github.com/juju/errors"

	"github.com/juju/juju/agent"
	"github.com/juju/juju/container"
	"github.com/juju/juju/container/kvm"
	"github.com/juju/juju/container/lxc"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/instance"
	"github.com/juju/juju/state"
	"github.com/juju/juju/state/api/params"
	apiprovisioner "github.com/juju/juju/state/api/provisioner"
	"github.com/juju/juju/state/api/watcher"
	"github.com/juju/juju/worker"
)

// ContainerSetup is a StringsWatchHandler that is notified when containers
// are created on the given machine. It will set up the machine to be able
// to create containers and start a suitable provisioner.
type ContainerSetup struct {
	runner              worker.Runner
	supportedContainers []instance.ContainerType
	provisioner         *apiprovisioner.State
	machine             *apiprovisioner.Machine
	config              agent.Config
	initLock            *fslock.Lock

	// Save the workerName so the worker thread can be stopped.
	workerName string
	// setupDone[containerType] is non zero if the container setup has been invoked
	// for that container type.
	setupDone map[instance.ContainerType]*int32
	// The number of provisioners started. Once all necessary provisioners have
	// been started, the container watcher can be stopped.
	numberProvisioners int32
}

// NewContainerSetupHandler returns a StringsWatchHandler which is notified when
// containers are created on the given machine.
func NewContainerSetupHandler(runner worker.Runner, workerName string, supportedContainers []instance.ContainerType,
	machine *apiprovisioner.Machine, provisioner *apiprovisioner.State,
	config agent.Config, initLock *fslock.Lock) worker.StringsWatchHandler {

	return &ContainerSetup{
		runner:              runner,
		machine:             machine,
		supportedContainers: supportedContainers,
		provisioner:         provisioner,
		config:              config,
		workerName:          workerName,
		initLock:            initLock,
	}
}

// SetUp is defined on the StringsWatchHandler interface.
func (cs *ContainerSetup) SetUp() (watcher watcher.StringsWatcher, err error) {
	// Set up the semaphores for each container type.
	cs.setupDone = make(map[instance.ContainerType]*int32, len(instance.ContainerTypes))
	for _, containerType := range instance.ContainerTypes {
		zero := int32(0)
		cs.setupDone[containerType] = &zero
	}
	// Listen to all container lifecycle events on our machine.
	if watcher, err = cs.machine.WatchAllContainers(); err != nil {
		return nil, err
	}
	return watcher, nil
}

// Handle is called whenever containers change on the machine being watched.
// Machines start out with no containers so the first time Handle is called,
// it will be because a container has been added.
func (cs *ContainerSetup) Handle(containerIds []string) (resultError error) {
	// Consume the initial watcher event.
	if len(containerIds) == 0 {
		return nil
	}

	logger.Infof("initial container setup with ids: %v", containerIds)
	for _, id := range containerIds {
		containerType := state.ContainerTypeFromId(id)
		// If this container type has been dealt with, do nothing.
		if atomic.LoadInt32(cs.setupDone[containerType]) != 0 {
			continue
		}
		if err := cs.initialiseAndStartProvisioner(containerType); err != nil {
			logger.Errorf("starting container provisioner for %v: %v", containerType, err)
			// Just because dealing with one type of container fails, we won't exit the entire
			// function because we still want to try and start other container types. So we
			// take note of and return the first such error.
			if resultError == nil {
				resultError = err
			}
		}
	}
	return resultError
}

func (cs *ContainerSetup) initialiseAndStartProvisioner(containerType instance.ContainerType) (resultError error) {
	// Flag that this container type has been handled.
	atomic.StoreInt32(cs.setupDone[containerType], 1)

	defer func() {
		if resultError != nil {
			logger.Warningf("not stopping machine agent container watcher due to error: %v", resultError)
			return
		}
		if atomic.AddInt32(&cs.numberProvisioners, 1) == int32(len(cs.supportedContainers)) {
			// We only care about the initial container creation.
			// This worker has done its job so stop it.
			// We do not expect there will be an error, and there's not much we can do anyway.
			if err := cs.runner.StopWorker(cs.workerName); err != nil {
				logger.Warningf("stopping machine agent container watcher: %v", err)
			}
		}
	}()

	logger.Debugf("setup and start provisioner for %s containers", containerType)
	if initialiser, broker, err := cs.getContainerArtifacts(containerType); err != nil {
		return errors.Annotate(err, "initialising container infrastructure on host machine")
	} else {
		if err := cs.runInitialiser(containerType, initialiser); err != nil {
			return errors.Annotate(err, "setting up container dependencies on host machine")
		}
		return StartProvisioner(cs.runner, containerType, cs.provisioner, cs.config, broker)
	}
}

// runInitialiser runs the container initialiser with the initialisation hook held.
func (cs *ContainerSetup) runInitialiser(containerType instance.ContainerType, initialiser container.Initialiser) error {
	logger.Debugf("running initialiser for %s containers", containerType)
	if err := cs.initLock.Lock(fmt.Sprintf("initialise-%s", containerType)); err != nil {
		return errors.Annotate(err, "failed to acquire initialization lock")
	}
	defer cs.initLock.Unlock()
	return initialiser.Initialise()
}

// TearDown is defined on the StringsWatchHandler interface.
func (cs *ContainerSetup) TearDown() error {
	// Nothing to do here.
	return nil
}

func (cs *ContainerSetup) getContainerArtifacts(containerType instance.ContainerType) (container.Initialiser, environs.InstanceBroker, error) {
	logger.Debugf("finding tools for %s containers", containerType)
	tools, err := cs.provisioner.Tools(cs.config.Tag())
	if err != nil {
		logger.Errorf("cannot get tools from machine for %s container", containerType)
		return nil, nil, err
	}
	var initialiser container.Initialiser
	var broker environs.InstanceBroker

	managerConfig, err := containerManagerConfig(containerType, cs.provisioner, cs.config)
	if err != nil {
		return nil, nil, err
	}

	switch containerType {
	case instance.LXC:
		series, err := cs.machine.Series()
		if err != nil {
			return nil, nil, err
		}

		initialiser = lxc.NewContainerInitialiser(series)
		broker, err = NewLxcBroker(cs.provisioner, tools, cs.config, managerConfig)
		if err != nil {
			return nil, nil, err
		}
	case instance.KVM:
		initialiser = kvm.NewContainerInitialiser()
		broker, err = NewKvmBroker(cs.provisioner, tools, cs.config, managerConfig)
		if err != nil {
			logger.Errorf("failed to create new kvm broker")
			return nil, nil, err
		}
	default:
		return nil, nil, fmt.Errorf("unknown container type: %v", containerType)
	}
	return initialiser, broker, nil
}

func containerManagerConfig(
	containerType instance.ContainerType,
	provisioner *apiprovisioner.State,
	agentConfig agent.Config,
) (container.ManagerConfig, error) {
	// Ask the provisioner for the container manager configuration.
	managerConfigResult, err := provisioner.ContainerManagerConfig(
		params.ContainerManagerConfigParams{Type: containerType},
	)
	if params.IsCodeNotImplemented(err) {
		// We currently don't support upgrading;
		// revert to the old configuration.
		managerConfigResult.ManagerConfig = container.ManagerConfig{container.ConfigName: "juju"}
	}
	if err != nil {
		return nil, err
	}
	// If a namespace is specified, that should instead be used as the config name.
	if namespace := agentConfig.Value(agent.Namespace); namespace != "" {
		managerConfigResult.ManagerConfig[container.ConfigName] = namespace
	}
	managerConfig := container.ManagerConfig(managerConfigResult.ManagerConfig)
	return managerConfig, nil
}

// Override for testing.
var StartProvisioner = startProvisionerWorker

// startProvisionerWorker kicks off a provisioner task responsible for creating containers
// of the specified type on the machine.
func startProvisionerWorker(runner worker.Runner, containerType instance.ContainerType,
	provisioner *apiprovisioner.State, config agent.Config, broker environs.InstanceBroker) error {

	workerName := fmt.Sprintf("%s-provisioner", containerType)
	// The provisioner task is created after a container record has already been added to the machine.
	// It will see that the container does not have an instance yet and create one.
	return runner.StartWorker(workerName, func() (worker.Worker, error) {
		return NewContainerProvisioner(containerType, provisioner, config, broker), nil
	})
}
<|endoftext|>


<commit_before>// Copyright 2018 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package git

import (
	"context"
	"encoding/hex"
	"fmt"
	"regexp"
	"time"

	"github.com/golang/protobuf/proto"

	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	"go.chromium.org/gae/service/memcache"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	gitpb "go.chromium.org/luci/common/proto/git"
	gitilespb "go.chromium.org/luci/common/proto/gitiles"
	"go.chromium.org/luci/common/tsmon/field"
	"go.chromium.org/luci/common/tsmon/metric"
	"go.chromium.org/luci/server/auth"
)

// LogOptions are options for Log function.
type LogOptions struct {
	Limit     int
	WithFiles bool
}

// Log implements Client interface.
func (p *implementation) Log(c context.Context, host, project, commitish string, inputOptions *LogOptions) (commits []*gitpb.Commit, err error) {
	defer func() { err = tagError(c, err) }()

	allowed, err := p.acls.IsAllowed(c, host, project)
	switch {
	case err != nil:
		return
	case !allowed:
		logging.Warningf(c, "%q not allowed for host %q project %q", auth.CurrentIdentity(c), host, project)
		err = status.Errorf(codes.NotFound, "not found")
		return
	}

	return p.log(c, host, project, commitish, "", inputOptions)
}

func (p *implementation) log(c context.Context, host, project, commitish, ancestor string, inputOptions *LogOptions) (commits []*gitpb.Commit, err error) {
	var opts LogOptions
	if inputOptions != nil {
		opts = *inputOptions
	}
	if opts.Limit <= 0 {
		opts.Limit = 50
	}

	remaining := opts.Limit // defined as (limit - len(commits))
	// add appends page to commits and updates remaining.
	// If page starts with commits' last commit, page's first commit
	// is skipped.
	add := func(page []*gitpb.Commit) {
		switch {
		case len(commits) == 0:
			commits = page
		case len(page) == 0:
			// Do nothing.
		default:
			if page[0].Id != commits[len(commits)-1].Id {
				panic(fmt.Sprintf(
					"pages are not contiguous; want page start %q, got %q",
					commits[len(commits)-1].Id, page[0].Id))
			}
			commits = append(commits, page[1:]...)
		}
		remaining = opts.Limit - len(commits)
	}

	req := &logReq{
		factory:   p,
		host:      host,
		project:   project,
		commitish: commitish,
		ancestor:  ancestor,
		withFiles: opts.WithFiles,
		min:       100,
	}

	var page []*gitpb.Commit
	for remaining > 100 {
		// We need to fetch >100 commits, but one logReq can handle only 100.
		// Call it in a loop.
		page, err = req.call(c)
		switch {
		case err != nil:
			return
		case len(page) < 100:
			// This can happen iff there are no more commits.
			add(page)
			return
		case len(page) > 100:
			panic("impossible: logReq.call() returned >100 commits")
		default:
			// There may be more commits.
			// Continue from the last fetched commit.
			req.commitish = page[len(page)-1].Id
			add(page)
			if remaining == 0 {
				panic("impossible: remaining reached 0")
			}
		}
	}

	// last page. One logReq.call() can handle the rest.
	req.min = remaining
	page, err = req.call(c)
	if err != nil {
		return
	}
	add(page)

	// we may end up with more than we were asked for because
	// gitReq's parameter is minimum, not limit.
	if len(commits) > opts.Limit {
		commits = commits[:opts.Limit]
	}
	return
}

// possible values for "result" field of logCounter metric below.
const (
	cacheHit        = "hit"
	cacheMiss       = "miss"
	cacheFailure    = "failure-cache"
	decodingFailure = "failure-decoding"
)

var logCounter = metric.NewCounter(
	"luci/milo/git/log/cache",
	"The number of hits we get in git.Log",
	nil,
	field.String("result"), // for possible value see consts above.
	field.String("host"),
	field.String("project"),
	field.String("ref"))

var gitHash = regexp.MustCompile("^[0-9a-fA-F]{40}$")

// logReq is the implementation of Log().
//
// Cached commits are keyed not only off of the requested commitish,
// but also ancestors, for example if received 100 ancestor
// commits C100, C99, .., C2, C1,
// cache with C99, C98, ... C1, too.
//
// The last byte of a cache value specifies how to interpret the rest:
//   0:       the rest is protobuf-marshalled gitilespb.LogResponse message
//   1-99:    the rest is byte id of a descendant commit that
//            the commit-in-cache-key is reachable from.
//            The last byte indicates the distance between the two commits.
//   100-255: reserved for future.
type logReq struct {
	factory   *implementation
	host      string
	project   string
	commitish string
	ancestor  string
	withFiles bool
	min       int // must be in [1..100]

	// fields below are set in call()

	commitishIsHash bool
	commitishEntry  memcache.Item
}

func (l *logReq) call(c context.Context) ([]*gitpb.Commit, error) {
	if l.min < 1 || l.min > 100 {
		panic(fmt.Sprintf("invalid min %d", l.min))
	}

	c = logging.SetFields(c, logging.Fields{
		"host":      l.host,
		"project":   l.project,
		"commitish": l.commitish,
	})

	l.commitishIsHash = gitHash.MatchString(l.commitish)
	l.commitishEntry = l.mkCache(c, l.commitish)
	if !l.commitishIsHash {
		// committish is not pinned, may move, so set a short expiration.
		l.commitishEntry.SetExpiration(30 * time.Second)
	}

	cacheResult := ""
	defer func() {
		ref := l.commitish
		if l.commitishIsHash {
			ref = "PINNED"
		}
		logCounter.Add(c, 1, cacheResult, l.host, l.project, ref)
	}()

	cacheResult, commits, ok := l.readCache(c)
	if ok {
		return commits, nil
	}

	// cache miss, cache failure or corrupted cache.
	// Call Gitiles.
	req := &gitilespb.LogRequest{
		Project:    l.project,
		Committish: l.commitish,
		PageSize:   100,
		TreeDiff:   l.withFiles,
	}
	if l.ancestor != "" {
		req.ExcludeAncestorsOf = l.ancestor
	}
	logging.Infof(c, "gitiles(%q).Log(%#v)", l.host, req)
	client, err := l.factory.gitilesClient(c, l.host)
	if err != nil {
		return nil, err
	}
	res, err := client.Log(c, req)
	if err != nil {
		return nil, errors.Annotate(err, "gitiles.Log").Err()
	}

	l.writeCache(c, res)
	return res.Log, nil
}

func (l *logReq) readCache(c context.Context) (cacheResult string, commits []*gitpb.Commit, ok bool) {
	e := l.commitishEntry
	dist := byte(0)
	maxDist := byte(100 - l.min)
	for {
		switch err := memcache.Get(c, e); {
		case err == memcache.ErrCacheMiss:
			logging.Infof(c, "cache miss")
			cacheResult = cacheMiss

		case err != nil:
			logging.WithError(err).Errorf(c, "cache failure")
			cacheResult = cacheFailure

		case len(e.Value()) == 0:
			logging.WithError(err).Errorf(c, "empty cache value at key %q", e.Key())
			cacheResult = decodingFailure
		default:
			n := len(e.Value())
			// see logReq for cache value format.
			data := e.Value()[:n-1]
			meta := e.Value()[n-1]
			switch {
			case meta == 0:
				var decoded gitilespb.LogResponse
				if err := proto.Unmarshal(data, &decoded); err != nil {
					logging.WithError(err).Errorf(c, "could not decode cached commits at key %q", e.Key())
					cacheResult = decodingFailure
				} else {
					cacheResult = cacheHit
					commits = decoded.Log[dist:]
					ok = true
				}

			case meta >= 100:
				logging.WithError(err).Errorf(c, "unexpected last byte %d in cache value at key %q", meta, e.Key())
				cacheResult = decodingFailure

			case dist+meta <= maxDist:
				// meta is distance, and the referenced cache is sufficient for reuse
				dist += meta
				descendant := hex.EncodeToString(data)
				logging.Debugf(c, "recursing into cache %s with distance %d", descendant, meta)
				e = l.mkCache(c, descendant)
				// cacheResult is not set => continue the loop.
			default:
				// TODO(nodir): if we need commits C200..C100, but we have
				// C200->C250 here (C250 entry has C250..C150), instead of
				// discarding cache, reuse C200..C150 from C250 and fetch
				// C150..C50.
				logging.Debugf(c, "distance at key %q is too large", e.Key())
				cacheResult = cacheMiss
			}
		}

		if cacheResult != "" {
			return
		}
	}
}

func (l *logReq) writeCache(c context.Context, res *gitilespb.LogResponse) {
	marshalled, err := proto.Marshal(res)
	if err != nil {
		logging.WithError(err).Errorf(c, "failed to marshal gitiles log %s", res)
		return
	}

	// see logReq comment for cache value format.
	l.commitishEntry.SetValue(append(marshalled, 0))

	// Cache entries to set.
	caches := make([]memcache.Item, 1, len(res.Log)+1)
	caches[0] = l.commitishEntry
	if !l.commitishIsHash && len(res.Log) > 0 {
		// cache with commit hash cache key too.
		e := l.mkCache(c, res.Log[0].Id)
		e.SetValue(l.commitishEntry.Value())
		caches = append(caches, e)
	}

	if len(res.Log) > 1 {
		// Also potentially cache with ancestors as cache keys.
		ancestorCaches := make([]memcache.Item, 0, len(res.Log)-1)
		for i := 1; i < len(res.Log) && len(res.Log[i-1].Parents) == 1; i++ {
			ancestorCaches = append(ancestorCaches, l.mkCache(c, res.Log[i].Id))
		}
		if err := memcache.Get(c, ancestorCaches...); err != nil {
			merr, ok := err.(errors.MultiError)
			if !ok {
				merr = errors.MultiError{err}
			}
			for i, ierr := range merr {
				e := ancestorCaches[i]
				if ierr != nil && ierr != memcache.ErrCacheMiss {
					logging.WithError(err).Errorf(c, "Failed to retrieve cache entry at %q", e.Key())
				}
				if ierr != nil {
					e.SetValue(nil)
				}
			}
		}

		topCommitID, err := hex.DecodeString(res.Log[0].Id)
		if err != nil {
			logging.WithError(err).Errorf(c, "commit id %q is not a valid hex", res.Log[0].Id)
		} else {
			for i, e := range ancestorCaches {
				dist := byte(i + 1)
				if v := e.Value(); len(v) > 0 && v[len(v)-1] <= dist {
					// This cache entry is not worse than what we can offer.
				} else {
					// We have data with a shorter distance.
					// see logReq comment for format of this cache value.
					v := make([]byte, len(topCommitID)+1)
					copy(v, topCommitID)
					v[len(v)-1] = dist
					e.SetValue(v)
					caches = append(caches, e)
				}
			}
		}
	}

	// This could be potentially improved by using CAS,
	// but it would significantly complicate this code.
	if err := memcache.Set(c, caches...); err != nil {
		logging.WithError(err).Errorf(c, "Failed to cache gitiles log")
	} else {
		logging.Debugf(c, "wrote %d entries", len(caches))
	}
}

func (l *logReq) mkCache(c context.Context, commitish string) memcache.Item {
	// note: better not to include limit in the cache key.
	key := fmt.Sprintf("git-log-%s|%s|%s|%s|%t", l.host, l.project, commitish,
		l.ancestor, l.withFiles)
	item := memcache.NewItem(c, key)
	// do not pollute memcache with items we probably won't need soon.
	item.SetExpiration(12 * time.Hour)
	return item
}
<commit_msg>[milo] Add metrics for gitiles latency<commit_after>// Copyright 2018 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package git

import (
	"context"
	"encoding/hex"
	"fmt"
	"regexp"
	"time"

	"github.com/golang/protobuf/proto"

	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	"go.chromium.org/gae/service/memcache"
	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	gitpb "go.chromium.org/luci/common/proto/git"
	gitilespb "go.chromium.org/luci/common/proto/gitiles"
	"go.chromium.org/luci/common/tsmon/distribution"
	"go.chromium.org/luci/common/tsmon/field"
	"go.chromium.org/luci/common/tsmon/metric"
	"go.chromium.org/luci/common/tsmon/types"
	"go.chromium.org/luci/server/auth"
)

// LogOptions are options for Log function.
type LogOptions struct {
	Limit     int
	WithFiles bool
}

// Log implements Client interface.
func (p *implementation) Log(c context.Context, host, project, commitish string, inputOptions *LogOptions) (commits []*gitpb.Commit, err error) {
	defer func() { err = tagError(c, err) }()

	allowed, err := p.acls.IsAllowed(c, host, project)
	switch {
	case err != nil:
		return
	case !allowed:
		logging.Warningf(c, "%q not allowed for host %q project %q", auth.CurrentIdentity(c), host, project)
		err = status.Errorf(codes.NotFound, "not found")
		return
	}

	return p.log(c, host, project, commitish, "", inputOptions)
}

func (p *implementation) log(c context.Context, host, project, commitish, ancestor string, inputOptions *LogOptions) (commits []*gitpb.Commit, err error) {
	var opts LogOptions
	if inputOptions != nil {
		opts = *inputOptions
	}
	if opts.Limit <= 0 {
		opts.Limit = 50
	}

	remaining := opts.Limit // defined as (limit - len(commits))
	// add appends page to commits and updates remaining.
	// If page starts with commits' last commit, page's first commit
	// is skipped.
	add := func(page []*gitpb.Commit) {
		switch {
		case len(commits) == 0:
			commits = page
		case len(page) == 0:
			// Do nothing.
		default:
			if page[0].Id != commits[len(commits)-1].Id {
				panic(fmt.Sprintf(
					"pages are not contiguous; want page start %q, got %q",
					commits[len(commits)-1].Id, page[0].Id))
			}
			commits = append(commits, page[1:]...)
		}
		remaining = opts.Limit - len(commits)
	}

	req := &logReq{
		factory:   p,
		host:      host,
		project:   project,
		commitish: commitish,
		ancestor:  ancestor,
		withFiles: opts.WithFiles,
		min:       100,
	}

	var page []*gitpb.Commit
	for remaining > 100 {
		// We need to fetch >100 commits, but one logReq can handle only 100.
		// Call it in a loop.
		page, err = req.call(c)
		switch {
		case err != nil:
			return
		case len(page) < 100:
			// This can happen iff there are no more commits.
			add(page)
			return
		case len(page) > 100:
			panic("impossible: logReq.call() returned >100 commits")
		default:
			// There may be more commits.
			// Continue from the last fetched commit.
			req.commitish = page[len(page)-1].Id
			add(page)
			if remaining == 0 {
				panic("impossible: remaining reached 0")
			}
		}
	}

	// last page. One logReq.call() can handle the rest.
	req.min = remaining
	page, err = req.call(c)
	if err != nil {
		return
	}
	add(page)

	// we may end up with more than we were asked for because
	// gitReq's parameter is minimum, not limit.
	if len(commits) > opts.Limit {
		commits = commits[:opts.Limit]
	}
	return
}

// possible values for "result" field of logCounter metric below.
const (
	cacheHit        = "hit"
	cacheMiss       = "miss"
	cacheFailure    = "failure-cache"
	decodingFailure = "failure-decoding"
)

var logCounter = metric.NewCounter(
	"luci/milo/git/log/cache",
	"The number of hits we get in git.Log",
	nil,
	field.String("result"), // for possible value see consts above.
	field.String("host"),
	field.String("project"),
	field.String("ref"))

var latencyMetric = metric.NewCumulativeDistribution(
	"luci/milo/git/log/latency",
	"Gitiles response latency",
	&types.MetricMetadata{Units: types.Milliseconds},
	distribution.DefaultBucketer,
	field.Bool("treeDiff"),
	field.String("host"),
	field.String("project"))

var gitHash = regexp.MustCompile("^[0-9a-fA-F]{40}$")

// logReq is the implementation of Log().
//
// Cached commits are keyed not only off of the requested commitish,
// but also ancestors, for example if received 100 ancestor
// commits C100, C99, .., C2, C1,
// cache with C99, C98, ... C1, too.
//
// The last byte of a cache value specifies how to interpret the rest:
//   0:       the rest is protobuf-marshalled gitilespb.LogResponse message
//   1-99:    the rest is byte id of a descendant commit that
//            the commit-in-cache-key is reachable from.
//            The last byte indicates the distance between the two commits.
//   100-255: reserved for future.
type logReq struct {
	factory   *implementation
	host      string
	project   string
	commitish string
	ancestor  string
	withFiles bool
	min       int // must be in [1..100]

	// fields below are set in call()

	commitishIsHash bool
	commitishEntry  memcache.Item
}

func (l *logReq) call(c context.Context) ([]*gitpb.Commit, error) {
	if l.min < 1 || l.min > 100 {
		panic(fmt.Sprintf("invalid min %d", l.min))
	}

	c = logging.SetFields(c, logging.Fields{
		"host":      l.host,
		"project":   l.project,
		"commitish": l.commitish,
	})

	l.commitishIsHash = gitHash.MatchString(l.commitish)
	l.commitishEntry = l.mkCache(c, l.commitish)
	if !l.commitishIsHash {
		// committish is not pinned, may move, so set a short expiration.
		l.commitishEntry.SetExpiration(30 * time.Second)
	}

	cacheResult := ""
	defer func() {
		ref := l.commitish
		if l.commitishIsHash {
			ref = "PINNED"
		}
		logCounter.Add(c, 1, cacheResult, l.host, l.project, ref)
	}()

	cacheResult, commits, ok := l.readCache(c)
	if ok {
		return commits, nil
	}

	// cache miss, cache failure or corrupted cache.
	// Call Gitiles.
	req := &gitilespb.LogRequest{
		Project:    l.project,
		Committish: l.commitish,
		PageSize:   100,
		TreeDiff:   l.withFiles,
	}
	if l.ancestor != "" {
		req.ExcludeAncestorsOf = l.ancestor
	}
	logging.Infof(c, "gitiles(%q).Log(%#v)", l.host, req)
	client, err := l.factory.gitilesClient(c, l.host)
	if err != nil {
		return nil, err
	}
	start := clock.Now(c)
	res, err := client.Log(c, req)
	latency := float64(clock.Now(c).Sub(start).Nanoseconds()) / 1000000.0
	logging.Debugf(c, "gitiles took %fms", latency)
	latencyMetric.Add(c, latency, l.withFiles, l.host, l.project)
	if err != nil {
		return nil, errors.Annotate(err, "gitiles.Log").Err()
	}

	l.writeCache(c, res)
	return res.Log, nil
}

func (l *logReq) readCache(c context.Context) (cacheResult string, commits []*gitpb.Commit, ok bool) {
	e := l.commitishEntry
	dist := byte(0)
	maxDist := byte(100 - l.min)
	for {
		switch err := memcache.Get(c, e); {
		case err == memcache.ErrCacheMiss:
			logging.Infof(c, "cache miss")
			cacheResult = cacheMiss

		case err != nil:
			logging.WithError(err).Errorf(c, "cache failure")
			cacheResult = cacheFailure

		case len(e.Value()) == 0:
			logging.WithError(err).Errorf(c, "empty cache value at key %q", e.Key())
			cacheResult = decodingFailure
		default:
			n := len(e.Value())
			// see logReq for cache value format.
			data := e.Value()[:n-1]
			meta := e.Value()[n-1]
			switch {
			case meta == 0:
				var decoded gitilespb.LogResponse
				if err := proto.Unmarshal(data, &decoded); err != nil {
					logging.WithError(err).Errorf(c, "could not decode cached commits at key %q", e.Key())
					cacheResult = decodingFailure
				} else {
					cacheResult = cacheHit
					commits = decoded.Log[dist:]
					ok = true
				}

			case meta >= 100:
				logging.WithError(err).Errorf(c, "unexpected last byte %d in cache value at key %q", meta, e.Key())
				cacheResult = decodingFailure

			case dist+meta <= maxDist:
				// meta is distance, and the referenced cache is sufficient for reuse
				dist += meta
				descendant := hex.EncodeToString(data)
				logging.Debugf(c, "recursing into cache %s with distance %d", descendant, meta)
				e = l.mkCache(c, descendant)
				// cacheResult is not set => continue the loop.
			default:
				// TODO(nodir): if we need commits C200..C100, but we have
				// C200->C250 here (C250 entry has C250..C150), instead of
				// discarding cache, reuse C200..C150 from C250 and fetch
				// C150..C50.
				logging.Debugf(c, "distance at key %q is too large", e.Key())
				cacheResult = cacheMiss
			}
		}

		if cacheResult != "" {
			return
		}
	}
}

func (l *logReq) writeCache(c context.Context, res *gitilespb.LogResponse) {
	marshalled, err := proto.Marshal(res)
	if err != nil {
		logging.WithError(err).Errorf(c, "failed to marshal gitiles log %s", res)
		return
	}

	// see logReq comment for cache value format.
	l.commitishEntry.SetValue(append(marshalled, 0))

	// Cache entries to set.
	caches := make([]memcache.Item, 1, len(res.Log)+1)
	caches[0] = l.commitishEntry
	if !l.commitishIsHash && len(res.Log) > 0 {
		// cache with commit hash cache key too.
		e := l.mkCache(c, res.Log[0].Id)
		e.SetValue(l.commitishEntry.Value())
		caches = append(caches, e)
	}

	if len(res.Log) > 1 {
		// Also potentially cache with ancestors as cache keys.
		ancestorCaches := make([]memcache.Item, 0, len(res.Log)-1)
		for i := 1; i < len(res.Log) && len(res.Log[i-1].Parents) == 1; i++ {
			ancestorCaches = append(ancestorCaches, l.mkCache(c, res.Log[i].Id))
		}
		if err := memcache.Get(c, ancestorCaches...); err != nil {
			merr, ok := err.(errors.MultiError)
			if !ok {
				merr = errors.MultiError{err}
			}
			for i, ierr := range merr {
				e := ancestorCaches[i]
				if ierr != nil && ierr != memcache.ErrCacheMiss {
					logging.WithError(err).Errorf(c, "Failed to retrieve cache entry at %q", e.Key())
				}
				if ierr != nil {
					e.SetValue(nil)
				}
			}
		}

		topCommitID, err := hex.DecodeString(res.Log[0].Id)
		if err != nil {
			logging.WithError(err).Errorf(c, "commit id %q is not a valid hex", res.Log[0].Id)
		} else {
			for i, e := range ancestorCaches {
				dist := byte(i + 1)
				if v := e.Value(); len(v) > 0 && v[len(v)-1] <= dist {
					// This cache entry is not worse than what we can offer.
				} else {
					// We have data with a shorter distance.
					// see logReq comment for format of this cache value.
					v := make([]byte, len(topCommitID)+1)
					copy(v, topCommitID)
					v[len(v)-1] = dist
					e.SetValue(v)
					caches = append(caches, e)
				}
			}
		}
	}

	// This could be potentially improved by using CAS,
	// but it would significantly complicate this code.
	if err := memcache.Set(c, caches...); err != nil {
		logging.WithError(err).Errorf(c, "Failed to cache gitiles log")
	} else {
		logging.Debugf(c, "wrote %d entries", len(caches))
	}
}

func (l *logReq) mkCache(c context.Context, commitish string) memcache.Item {
	// note: better not to include limit in the cache key.
	key := fmt.Sprintf("git-log-%s|%s|%s|%s|%t", l.host, l.project, commitish,
		l.ancestor, l.withFiles)
	item := memcache.NewItem(c, key)
	// do not pollute memcache with items we probably won't need soon.
	item.SetExpiration(12 * time.Hour)
	return item
}
<|endoftext|>


<commit_before>package frontend

import (
	"bytes"
	"encoding/base64"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/google/uuid"
	"github.com/gorilla/mux"

	gw "github.com/cvmfs/gateway/internal/gateway"
	be "github.com/cvmfs/gateway/internal/gateway/backend"
)

// MakeAuthzMiddleware returns an HMAC authorization middleware for use with the gorilla/mux server
func MakeAuthzMiddleware(ac *be.AccessConfig) mux.MiddlewareFunc {
	return mux.MiddlewareFunc(func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
			// GET requests do not need authorization
			if req.Method == "GET" {
				next.ServeHTTP(w, req)
				return
			}

			reqID, _ := req.Context().Value(idKey).(uuid.UUID)
			tokens := strings.Split(req.Header.Get("Authorization"), " ")
			if len(tokens) != 2 {
				gw.Log.Error().
					Str("component", "http").
					Str("req_id", reqID.String()).
					Msg("missing tokens in authorization header")
				replyJSON(&reqID, w, message{"status": "error", "reason": "invalid_hmac"})
				return
			}

			keyID := tokens[0]
			HMAC, err := base64.StdEncoding.DecodeString(tokens[1])
			if err != nil {
				gw.Log.Error().
					Str("component", "http").
					Str("req_id", reqID.String()).
					Err(err).Msg("could not base64 decode HMAC")
				replyJSON(&reqID, w, message{"status": "error", "reason": "invalid_hmac"})
				return
			}

			secret := ac.GetSecret(keyID)
			if len(secret) == 0 {
				gw.Log.Error().
					Str("component", "http").
					Str("req_id", reqID.String()).
					Msg("invalid key ID specified")
				replyJSON(&reqID, w, message{"status": "error", "reason": "invalid_hmac"})
				return
			}

			// Different parts of the request are used to compute then HMAC, depending
			// in HTTP method and route

			var HMACInput []byte
			if strings.HasPrefix(req.URL.Path, APIRoot+"/repos") {
				// /repos request, use the path component of the URL to compute HMAC
				HMACInput = []byte(req.URL.Path)
			} else if strings.HasPrefix(req.URL.Path, APIRoot+"/leases") {
				token, _ := mux.Vars(req)["token"]
				if token != "" {
					// For commit/drop lease requests use the token to compute HMAC
					HMACInput = []byte(token)
				} else {
					// For new lease request used the request body to compute HMAC
					HMACInput, err = ioutil.ReadAll(req.Body)
					if err != nil {
						httpWrapError(&reqID, err, "could not read request body", w, http.StatusBadRequest)
						return
					}
					// Body needs to be read again in the next handler, reset it
					// using a copy of the original body
					bodyCopy := ioutil.NopCloser(bytes.NewReader(HMACInput))
					req.Body.Close()
					req.Body = bodyCopy
				}
			}
			if !CheckHMAC(HMACInput, HMAC, secret) {
				gw.Log.Error().
					Str("component", "http").
					Str("req_id", reqID.String()).
					Msg("invalid HMAC")
				replyJSON(&reqID, w, message{"status": "error", "reason": "invalid_hmac"})
				return
			}
			next.ServeHTTP(w, req)
		})
	})
}
<commit_msg>Implemented authorization checks for /payloads endpoint<commit_after>package frontend

import (
	"bytes"
	"encoding/base64"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"

	"github.com/google/uuid"
	"github.com/gorilla/mux"

	gw "github.com/cvmfs/gateway/internal/gateway"
	be "github.com/cvmfs/gateway/internal/gateway/backend"
)

// MakeAuthzMiddleware returns an HMAC authorization middleware for use with the gorilla/mux server
func MakeAuthzMiddleware(ac *be.AccessConfig) mux.MiddlewareFunc {
	return mux.MiddlewareFunc(func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
			// GET requests do not need authorization
			if req.Method == "GET" {
				next.ServeHTTP(w, req)
				return
			}

			reqID, _ := req.Context().Value(idKey).(uuid.UUID)
			tokens := strings.Split(req.Header.Get("Authorization"), " ")
			if len(tokens) != 2 {
				gw.Log.Error().
					Str("component", "http").
					Str("req_id", reqID.String()).
					Msg("missing tokens in authorization header")
				replyJSON(&reqID, w, message{"status": "error", "reason": "invalid_hmac"})
				return
			}

			keyID := tokens[0]
			HMAC, err := base64.StdEncoding.DecodeString(tokens[1])
			if err != nil {
				gw.Log.Error().
					Str("component", "http").
					Str("req_id", reqID.String()).
					Err(err).Msg("could not base64 decode HMAC")
				replyJSON(&reqID, w, message{"status": "error", "reason": "invalid_hmac"})
				return
			}

			secret := ac.GetSecret(keyID)
			if len(secret) == 0 {
				gw.Log.Error().
					Str("component", "http").
					Str("req_id", reqID.String()).
					Msg("invalid key ID specified")
				replyJSON(&reqID, w, message{"status": "error", "reason": "invalid_hmac"})
				return
			}

			// Different parts of the request are used to compute then HMAC, depending
			// in HTTP method and route

			var HMACInput []byte
			if strings.HasPrefix(req.URL.Path, APIRoot+"/leases") {
				token, _ := mux.Vars(req)["token"]
				if token != "" {
					// For commit/drop lease requests use the token to compute HMAC
					HMACInput = []byte(token)
				} else {
					// For new lease request used the request body to compute HMAC
					HMACInput, err = ioutil.ReadAll(req.Body)
					if err != nil {
						httpWrapError(&reqID, err, "could not read request body", w, http.StatusInternalServerError)
						return
					}
					// Body needs to be read again in the next handler, reset it
					// using a copy of the original body
					bodyCopy := ioutil.NopCloser(bytes.NewReader(HMACInput))
					req.Body.Close()
					req.Body = bodyCopy
				}
			} else if strings.HasPrefix(req.URL.Path, APIRoot+"/payloads") {
				token, _ := mux.Vars(req)["token"]
				if token != "" {
					// For the new style of payload submission requests, use the token to compute HMAC
					HMACInput = []byte(token)
				} else {
					// For legacy payload submission requests, the JSON msg at the beginning of the body
					// is used to compute the HMAC
					msgSize, err := strconv.Atoi(req.Header.Get("message-size"))
					if err != nil {
						httpWrapError(&reqID, err, "missing message-size header", w, http.StatusBadRequest)
					}
					body, err := ioutil.ReadAll(req.Body)
					if err != nil {
						httpWrapError(&reqID, err, "could not read request body", w, http.StatusInternalServerError)
					}
					HMACInput = body[:msgSize]
					req.Body.Close()
					req.Body = ioutil.NopCloser(bytes.NewReader(body))
				}
			}

			if !CheckHMAC(HMACInput, HMAC, secret) {
				gw.Log.Error().
					Str("component", "http").
					Str("req_id", reqID.String()).
					Msg("invalid HMAC")
				replyJSON(&reqID, w, message{"status": "error", "reason": "invalid_hmac"})
				return
			}
			next.ServeHTTP(w, req)
		})
	})
}
<|endoftext|>


<commit_before>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package source

import (
	"context"
	"fmt"
	"go/format"
	"go/types"
	"strings"

	"golang.org/x/tools/go/ast/astutil"
	"golang.org/x/tools/internal/lsp/protocol"
)

// FillStruct completes all of targeted struct's fields with their default values.
func FillStruct(ctx context.Context, snapshot Snapshot, fh FileHandle, protoRng protocol.Range) ([]protocol.CodeAction, error) {

	pkg, pgh, err := getParsedFile(ctx, snapshot, fh, NarrowestPackageHandle)
	if err != nil {
		return nil, fmt.Errorf("getting file for struct fill code action: %v", err)
	}
	file, src, m, _, err := pgh.Cached()
	if err != nil {
		return nil, err
	}
	spn, err := m.PointSpan(protoRng.Start)
	if err != nil {
		return nil, err
	}
	spanRng, err := spn.Range(m.Converter)
	if err != nil {
		return nil, err
	}
	path, _ := astutil.PathEnclosingInterval(file, spanRng.Start, spanRng.End)
	if path == nil {
		return nil, nil
	}

	ecl := enclosingCompositeLiteral(path, spanRng.Start, pkg.GetTypesInfo())
	if ecl == nil || !ecl.isStruct() {
		return nil, nil
	}

	// If in F{ Bar<> : V} or anywhere in F{Bar : V, ...}
	// we should not fill the struct.
	if ecl.inKey || len(ecl.cl.Elts) != 0 {
		return nil, nil
	}

	var codeActions []protocol.CodeAction
	qfFunc := qualifier(file, pkg.GetTypes(), pkg.GetTypesInfo())
	switch obj := ecl.clType.(type) {
	case *types.Struct:
		fieldCount := obj.NumFields()
		if fieldCount == 0 {
			return nil, nil
		}
		var fieldSourceCode strings.Builder
		for i := 0; i < fieldCount; i++ {
			field := obj.Field(i)
			// Ignore fields that are not accessible in the current package.
			if field.Pkg() != nil && field.Pkg() != pkg.GetTypes() && !field.Exported() {
				continue
			}

			label := field.Name()
			value := formatZeroValue(field.Type(), qfFunc)
			fieldSourceCode.WriteString("\n")
			fieldSourceCode.WriteString(label)
			fieldSourceCode.WriteString(" : ")
			fieldSourceCode.WriteString(value)
			fieldSourceCode.WriteString(",")
		}

		if fieldSourceCode.Len() == 0 {
			return nil, nil
		}

		fieldSourceCode.WriteString("\n")

		// the range of all text between '<>', inclusive. E.g.  {<> ...  <}>
		mappedRange := newMappedRange(snapshot.View().Session().Cache().FileSet(), m, ecl.cl.Lbrace, ecl.cl.Rbrace+1)
		protoRange, err := mappedRange.Range()
		if err != nil {
			return nil, err
		}
		// consider formatting from the first character of the line the lbrace is on.
		// ToOffset is 1-based
		beginOffset, err := m.Converter.ToOffset(int(protoRange.Start.Line)+1, 1)
		if err != nil {
			return nil, err
		}

		endOffset, err := m.Converter.ToOffset(int(protoRange.Start.Line)+1, int(protoRange.Start.Character)+1)
		if err != nil {
			return nil, err
		}

		// An increment to make sure the lbrace is included in the slice.
		endOffset++
		// Append the edits. Then append the closing brace.
		var newSourceCode strings.Builder
		newSourceCode.Grow(endOffset - beginOffset + fieldSourceCode.Len() + 1)
		newSourceCode.WriteString(string(src[beginOffset:endOffset]))
		newSourceCode.WriteString(fieldSourceCode.String())
		newSourceCode.WriteString("}")

		buf, err := format.Source([]byte(newSourceCode.String()))
		if err != nil {
			return nil, err
		}

		// it is guaranteed that a left brace exists.
		var edit = string(buf[strings.IndexByte(string(buf), '{'):])

		codeActions = append(codeActions, protocol.CodeAction{
			Title: "Fill struct",
			Kind:  protocol.RefactorRewrite,
			Edit: protocol.WorkspaceEdit{
				DocumentChanges: []protocol.TextDocumentEdit{
					{
						TextDocument: protocol.VersionedTextDocumentIdentifier{
							Version: fh.Identity().Version,
							TextDocumentIdentifier: protocol.TextDocumentIdentifier{
								URI: protocol.URIFromSpanURI(fh.Identity().URI),
							},
						},
						Edits: []protocol.TextEdit{
							{
								Range:   protoRange,
								NewText: edit,
							},
						},
					},
				},
			},
		})
	}

	return codeActions, nil
}
<commit_msg>internal/lsp/source: format fill_struct.go<commit_after>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package source

import (
	"context"
	"fmt"
	"go/format"
	"go/types"
	"strings"

	"golang.org/x/tools/go/ast/astutil"
	"golang.org/x/tools/internal/lsp/protocol"
)

// FillStruct completes all of targeted struct's fields with their default values.
func FillStruct(ctx context.Context, snapshot Snapshot, fh FileHandle, protoRng protocol.Range) ([]protocol.CodeAction, error) {

	pkg, pgh, err := getParsedFile(ctx, snapshot, fh, NarrowestPackageHandle)
	if err != nil {
		return nil, fmt.Errorf("getting file for struct fill code action: %v", err)
	}
	file, src, m, _, err := pgh.Cached()
	if err != nil {
		return nil, err
	}
	spn, err := m.PointSpan(protoRng.Start)
	if err != nil {
		return nil, err
	}
	spanRng, err := spn.Range(m.Converter)
	if err != nil {
		return nil, err
	}
	path, _ := astutil.PathEnclosingInterval(file, spanRng.Start, spanRng.End)
	if path == nil {
		return nil, nil
	}

	ecl := enclosingCompositeLiteral(path, spanRng.Start, pkg.GetTypesInfo())
	if ecl == nil || !ecl.isStruct() {
		return nil, nil
	}

	// If in F{ Bar<> : V} or anywhere in F{Bar : V, ...}
	// we should not fill the struct.
	if ecl.inKey || len(ecl.cl.Elts) != 0 {
		return nil, nil
	}

	var codeActions []protocol.CodeAction
	qfFunc := qualifier(file, pkg.GetTypes(), pkg.GetTypesInfo())
	switch obj := ecl.clType.(type) {
	case *types.Struct:
		fieldCount := obj.NumFields()
		if fieldCount == 0 {
			return nil, nil
		}
		var fieldSourceCode strings.Builder
		for i := 0; i < fieldCount; i++ {
			field := obj.Field(i)
			// Ignore fields that are not accessible in the current package.
			if field.Pkg() != nil && field.Pkg() != pkg.GetTypes() && !field.Exported() {
				continue
			}

			label := field.Name()
			value := formatZeroValue(field.Type(), qfFunc)
			fieldSourceCode.WriteString("\n")
			fieldSourceCode.WriteString(label)
			fieldSourceCode.WriteString(" : ")
			fieldSourceCode.WriteString(value)
			fieldSourceCode.WriteString(",")
		}

		if fieldSourceCode.Len() == 0 {
			return nil, nil
		}

		fieldSourceCode.WriteString("\n")

		// the range of all text between '<>', inclusive. E.g.  {<> ...  <}>
		mappedRange := newMappedRange(snapshot.View().Session().Cache().FileSet(), m, ecl.cl.Lbrace, ecl.cl.Rbrace+1)
		protoRange, err := mappedRange.Range()
		if err != nil {
			return nil, err
		}
		// consider formatting from the first character of the line the lbrace is on.
		// ToOffset is 1-based
		beginOffset, err := m.Converter.ToOffset(int(protoRange.Start.Line)+1, 1)
		if err != nil {
			return nil, err
		}

		endOffset, err := m.Converter.ToOffset(int(protoRange.Start.Line)+1, int(protoRange.Start.Character)+1)
		if err != nil {
			return nil, err
		}

		// An increment to make sure the lbrace is included in the slice.
		endOffset++
		// Append the edits. Then append the closing brace.
		var newSourceCode strings.Builder
		newSourceCode.Grow(endOffset - beginOffset + fieldSourceCode.Len() + 1)
		newSourceCode.WriteString(string(src[beginOffset:endOffset]))
		newSourceCode.WriteString(fieldSourceCode.String())
		newSourceCode.WriteString("}")

		buf, err := format.Source([]byte(newSourceCode.String()))
		if err != nil {
			return nil, err
		}

		// it is guaranteed that a left brace exists.
		var edit = string(buf[strings.IndexByte(string(buf), '{'):])

		codeActions = append(codeActions, protocol.CodeAction{
			Title: "Fill struct",
			Kind:  protocol.RefactorRewrite,
			Edit: protocol.WorkspaceEdit{
				DocumentChanges: []protocol.TextDocumentEdit{
					{
						TextDocument: protocol.VersionedTextDocumentIdentifier{
							Version: fh.Identity().Version,
							TextDocumentIdentifier: protocol.TextDocumentIdentifier{
								URI: protocol.URIFromSpanURI(fh.Identity().URI),
							},
						},
						Edits: []protocol.TextEdit{
							{
								Range:   protoRange,
								NewText: edit,
							},
						},
					},
				},
			},
		})
	}

	return codeActions, nil
}
<|endoftext|>