<commit_before>package api

import (
	"bytes"
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"

	"github.com/apex/log"
	"github.com/juju/errors"
)

var (
	// APIUrl represents Online's endpoint
	APIUrl = "https://api.online.net/api/v1"
)

// OnlineAPI is used to communicate with Online API
type OnlineAPI struct {
	client    *http.Client
	userAgent string
	verbose   bool
	cache     *cache
}

// NewC14API returns a new API
func NewC14API(client *http.Client, userAgent string, verbose bool) (api *OnlineAPI) {
	api = &OnlineAPI{
		client:    client,
		userAgent: userAgent,
		verbose:   verbose,
		cache:     NewCache(),
	}
	return
}

func (o *OnlineAPI) response(method, uri string, content io.Reader) (resp *http.Response, err error) {
	var (
		req *http.Request
	)

	req, err = http.NewRequest(method, uri, content)
	if err != nil {
		err = errors.Annotatef(err, "response %s %s", method, uri)
		return
	}
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("User-Agent", o.userAgent)

	// curl, err := http2curl.GetCurlCommand(req)
	// if err != nil {
	// 	return nil, err
	// }
	if o.verbose {
		dump, _ := httputil.DumpRequest(req, true)
		log.Debugf("%v", string(dump))
	} else {
		log.Debugf("[%s]: %v", method, uri)
	}
	resp, err = o.client.Do(req)
	return
}

func (o *OnlineAPI) getWrapper(uri string, export interface{}) (err error) {
	var (
		resp *http.Response
		body []byte
	)

	resp, err = o.response("GET", uri, nil)
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		err = errors.Annotatef(err, "Unable to get %s", uri)
		return
	}

	if body, err = o.handleHTTPError([]int{200}, resp); err != nil {
		return
	}
	err = json.Unmarshal(body, export)
	return
}

func (o *OnlineAPI) deleteWrapper(uri string) (err error) {
	var (
		resp *http.Response
	)

	resp, err = o.response("DELETE", uri, nil)
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		err = errors.Annotatef(err, "Unable to delete %s", uri)
		return
	}

	if _, err = o.handleHTTPError([]int{204}, resp); err != nil {
		return
	}
	return
}

func (o *OnlineAPI) postWrapper(uri string, content interface{}) (body []byte, err error) {
	var (
		resp    *http.Response
		payload = new(bytes.Buffer)
	)

	encoder := json.NewEncoder(payload)
	if err = encoder.Encode(content); err != nil {
		return
	}
	resp, err = o.response("POST", uri, payload)
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		err = errors.Annotatef(err, "Unable to post %s", uri)
		return
	}
	body, err = o.handleHTTPError([]int{201}, resp)
	return
}

func (o *OnlineAPI) handleHTTPError(goodStatusCode []int, resp *http.Response) (content []byte, err error) {
	content, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}

	if o.verbose {
		dump, _ := httputil.DumpResponse(resp, true)
		log.Debugf("%v", string(dump))
	} else {
		log.Debugf("[Response]: [%v] %v", resp.StatusCode, string(content))
	}

	if resp.StatusCode >= 500 {
		err = errors.Errorf("[%v] %v", resp.StatusCode, string(content))
		return
	}
	good := false
	for _, code := range goodStatusCode {
		if code == resp.StatusCode {
			good = true
		}
	}
	if !good {
		var why OnlineError

		if err = json.Unmarshal(content, &why); err != nil {
			return
		}
		why.StatusCode = resp.StatusCode
		err = why
	}
	return
}
<commit_msg>PostWrapper takes GoodStatusCode array<commit_after>package api

import (
	"bytes"
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"

	"github.com/apex/log"
	"github.com/juju/errors"
)

var (
	// APIUrl represents Online's endpoint
	APIUrl = "https://api.online.net/api/v1"
)

// OnlineAPI is used to communicate with Online API
type OnlineAPI struct {
	client    *http.Client
	userAgent string
	verbose   bool
	cache     *cache
}

// NewC14API returns a new API
func NewC14API(client *http.Client, userAgent string, verbose bool) (api *OnlineAPI) {
	api = &OnlineAPI{
		client:    client,
		userAgent: userAgent,
		verbose:   verbose,
		cache:     NewCache(),
	}
	return
}

func (o *OnlineAPI) response(method, uri string, content io.Reader) (resp *http.Response, err error) {
	var (
		req *http.Request
	)

	req, err = http.NewRequest(method, uri, content)
	if err != nil {
		err = errors.Annotatef(err, "response %s %s", method, uri)
		return
	}
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("User-Agent", o.userAgent)

	// curl, err := http2curl.GetCurlCommand(req)
	// if err != nil {
	// 	return nil, err
	// }
	if o.verbose {
		dump, _ := httputil.DumpRequest(req, true)
		log.Debugf("%v", string(dump))
	} else {
		log.Debugf("[%s]: %v", method, uri)
	}
	resp, err = o.client.Do(req)
	return
}

func (o *OnlineAPI) getWrapper(uri string, export interface{}) (err error) {
	var (
		resp *http.Response
		body []byte
	)

	resp, err = o.response("GET", uri, nil)
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		err = errors.Annotatef(err, "Unable to get %s", uri)
		return
	}

	if body, err = o.handleHTTPError([]int{200}, resp); err != nil {
		return
	}
	err = json.Unmarshal(body, export)
	return
}

func (o *OnlineAPI) deleteWrapper(uri string) (err error) {
	var (
		resp *http.Response
	)

	resp, err = o.response("DELETE", uri, nil)
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		err = errors.Annotatef(err, "Unable to delete %s", uri)
		return
	}

	if _, err = o.handleHTTPError([]int{204}, resp); err != nil {
		return
	}
	return
}

func (o *OnlineAPI) postWrapper(uri string, content interface{}, goodStatusCode ...[]int) (body []byte, err error) {
	var (
		resp    *http.Response
		payload = new(bytes.Buffer)
	)

	encoder := json.NewEncoder(payload)
	if content != nil {
		if err = encoder.Encode(content); err != nil {
			return
		}
	}
	resp, err = o.response("POST", uri, payload)
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		err = errors.Annotatef(err, "Unable to post %s", uri)
		return
	}
	goodStatus := []int{201}
	if len(goodStatusCode) > 0 {
		goodStatus = goodStatusCode[0]
	}
	body, err = o.handleHTTPError(goodStatus, resp)
	return
}

func (o *OnlineAPI) handleHTTPError(goodStatusCode []int, resp *http.Response) (content []byte, err error) {
	content, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}

	if o.verbose {
		dump, _ := httputil.DumpResponse(resp, true)
		log.Debugf("%v", string(dump))
	} else {
		log.Debugf("[Response]: [%v] %v", resp.StatusCode, string(content))
	}

	if resp.StatusCode >= 500 {
		err = errors.Errorf("[%v] %v", resp.StatusCode, string(content))
		return
	}
	good := false
	for _, code := range goodStatusCode {
		if code == resp.StatusCode {
			good = true
		}
	}
	if !good {
		var why OnlineError

		if err = json.Unmarshal(content, &why); err != nil {
			return
		}
		why.StatusCode = resp.StatusCode
		err = why
	}
	return
}
<|endoftext|>


<commit_before>// +build !windows

package main

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/go-check/check"
)

// #16665
func (s *DockerSuite) TestInspectApiCpusetInConfigPre120(c *check.C) {
	testRequires(c, DaemonIsLinux)
	testRequires(c, cgroupCpuset)

	name := "cpusetinconfig-pre120"
	dockerCmd(c, "run", "--name", name, "--cpuset-cpus", "0", "busybox", "true")

	status, body, err := sockRequest("GET", fmt.Sprintf("/v1.19/containers/%s/json", name), nil)
	c.Assert(status, check.Equals, http.StatusOK)
	c.Assert(err, check.IsNil)

	var inspectJSON map[string]interface{}
	if err = json.Unmarshal(body, &inspectJSON); err != nil {
		c.Fatalf("unable to unmarshal body for version 1.19: %v", err)
	}

	config, ok := inspectJSON["Config"]
	if !ok {
		c.Fatal("Unable to find 'Config'")
	}
	cfg := config.(map[string]interface{})
	if _, ok := cfg["Cpuset"]; !ok {
		c.Fatal("Api version 1.19 expected to include Cpuset in 'Config'")
	}
}
<commit_msg>Checkers on docker_api_inspect_unix_test.go<commit_after>// +build !windows

package main

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/docker/docker/pkg/integration/checker"
	"github.com/go-check/check"
)

// #16665
func (s *DockerSuite) TestInspectApiCpusetInConfigPre120(c *check.C) {
	testRequires(c, DaemonIsLinux)
	testRequires(c, cgroupCpuset)

	name := "cpusetinconfig-pre120"
	dockerCmd(c, "run", "--name", name, "--cpuset-cpus", "0", "busybox", "true")

	status, body, err := sockRequest("GET", fmt.Sprintf("/v1.19/containers/%s/json", name), nil)
	c.Assert(status, check.Equals, http.StatusOK)
	c.Assert(err, check.IsNil)

	var inspectJSON map[string]interface{}
	err = json.Unmarshal(body, &inspectJSON)
	c.Assert(err, checker.IsNil, check.Commentf("unable to unmarshal body for version 1.19"))

	config, ok := inspectJSON["Config"]
	c.Assert(ok, checker.True, check.Commentf("Unable to find 'Config'"))
	cfg := config.(map[string]interface{})
	_, ok = cfg["Cpuset"]
	c.Assert(ok, checker.True, check.Commentf("Api version 1.19 expected to include Cpuset in 'Config'"))
}
<|endoftext|>


<commit_before>// Copyright 2011 Kevin Bulusek. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gameboy

import (
	"⚛sdl"
	"time"
)

const (
	modeHBlank = byte(iota)
	modeVBlank
	modeOAM
	modeVRAM
)

const (
	tilesX   = 20
	tilesY   = 18
	tileW    = 8
	tileH    = 8
	mapW     = 32
	mapH     = 32
	displayW = tileW * tilesX
	displayH = tileH * tilesY

	hblankTicks = 204 / 4
	vblankTicks = 4560 / 4
	oamTicks    = 80 / 4
	vramTicks   = 172 / 4

	scanlineTicks = oamTicks + vramTicks + hblankTicks
	refreshTicks  = scanlineTicks*displayH + vblankTicks
)

type display struct {
	*memory
	*sdl.Surface
	pal       [4]uint32
	frameTime int64
	screenW   int
	screenH   int

	clock int

	// LCDC flags
	enable       bool
	windowMap    bool
	windowEnable bool
	tileData     bool
	bgMap        bool
	spriteSize   bool
	spriteEnable bool
	bgEnable     bool

	// STAT flags
	lycInterrupt    bool
	oamInterrupt    bool
	vblankInterrupt bool
	hblankInterrupt bool
	mode            byte

	// LCD registers
	ly       byte
	scy, scx byte
	wy, wx   byte

	bgp [4]byte
	obp [2][4]byte

	// Scanlines are rendered to here first, and then drawn to the
	// display - rather than each layer accessing the display
	// separately (which is slow).
	lineBuf [displayW]byte

	// When rendering a scanline this is zeroed out, then
	// logical-ORed with the pixels from the BG and window. This
	// is then used to lookup which pixels can be painted in
	// sprites that are to be obscured by the background layers.
	oamLineMask [displayW]byte
}

func newDisplay(m *memory) *display {
	sdl.WM_SetCaption(m.rom.title(), "")
	lcd := display{memory: m}
	lcd.screenW = displayW * m.config.Scale
	lcd.screenH = displayH * m.config.Scale
	lcd.Surface = sdl.SetVideoMode(lcd.screenW, lcd.screenH, 0,
		sdl.DOUBLEBUF)
	lcd.pal[0] = sdl.MapRGBA(lcd.Format, 0x9B, 0xBC, 0x0F, 0)
	lcd.pal[1] = sdl.MapRGBA(lcd.Format, 0x8B, 0xAC, 0x0F, 0)
	lcd.pal[2] = sdl.MapRGBA(lcd.Format, 0x30, 0x62, 0x30, 0)
	lcd.pal[3] = sdl.MapRGBA(lcd.Format, 0x0F, 0x38, 0x0F, 0)
	lcd.FillRect(nil, lcd.pal[0])
	lcd.Flip()
	lcd.frameTime = time.Nanoseconds()
	return &lcd
}

func (lcd *display) step(t int) {
	lcd.clock += t
	if lcd.clock >= refreshTicks {
		lcd.clock -= refreshTicks
	}

	lcd.ly = byte(lcd.clock / scanlineTicks)
	lcd.hram[portLY-0xFF00] = lcd.ly
	mode := calcMode(lcd.clock, lcd.ly)
	if mode == lcd.mode {
		return
	}
	lcd.mode = mode

	stat := lcd.readPort(portSTAT)&^3 | mode
	irq := lcd.readPort(portIF)

	switch mode {
	case modeOAM:
		if lcd.oamInterrupt {
			lcd.writePort(portIF, irq|0x02)
		}
		if lyc := lcd.readPort(portLYC); lcd.ly-1 == lyc {
			stat |= 0x04
			if lcd.lycInterrupt {
				lcd.writePort(portIF, irq|0x02)
			}
		} else {
			stat &^= 0x04
		}
	case modeVRAM:
		if lcd.enable {
			lcd.scanline()
		}
	case modeHBlank:
		if lcd.hblankInterrupt {
			lcd.writePort(portIF, irq|0x02)
		}
	case modeVBlank:
		if lcd.vblankInterrupt {
			irq |= 0x02
		}
		lcd.writePort(portIF, irq|0x01)
		lcd.Flip()

		// while audio is playing, we let it control the
		// emulation speed
		if !lcd.audio.enable {
			lcd.delay()
		}
	}

	lcd.writePort(portSTAT, stat)
}

func (lcd *display) delay() {
	now := time.Nanoseconds()
	delta := now - lcd.frameTime
	target := 16742706 - delta
	if target > 0 {
		time.Sleep(target)
	}
	lcd.frameTime = time.Nanoseconds()
}

func (lcd *display) scanline() {
	for i := 0; i < displayW; i++ {
		lcd.oamLineMask[i] = 0
		lcd.lineBuf[i] = 0
	}

	if lcd.bgEnable {
		lcd.mapline(lcd.bgMap, byte(0), lcd.scx, lcd.scy)
	}

	if lcd.windowEnable {
		if lcd.wx < 167 && lcd.wy < 144 && lcd.ly >= lcd.wy {
			x := int(lcd.wx) - 7
			xoff := -x
			if x < 0 {
				x = 0
			}
			lcd.mapline(lcd.windowMap, byte(x),
				byte(xoff), byte(-lcd.wy))
		}
	}

	if lcd.spriteEnable {
		lcd.oamline()
	}

	lcd.flushline()
}

func (lcd *display) mapline(map1 bool, x, xoff, yoff byte) {
	y := lcd.ly + yoff
	for ; x < displayW; x++ {
		b := lcd.mapAt(map1, int(x+xoff), int(y))
		lcd.oamLineMask[x] |= b
		lcd.lineBuf[x] = lcd.bgp[b]
	}
}

func (lcd *display) flushline() {
	// Do some simple run-length counting to reduce the number of
	// FillRect calls we need to make.
	scale := uint16(lcd.config.Scale)
	r := &sdl.Rect{0, int16(lcd.ly) * int16(scale), scale, scale}
	cur := lcd.lineBuf[0]
	for x := 1; x < displayW; x++ {
		b := lcd.lineBuf[x]
		if b != cur {
			lcd.FillRect(r, lcd.pal[cur])
			cur = b
			r.X = int16(x) * int16(scale)
			r.W = scale
		} else {
			r.W += scale
		}
	}
	lcd.FillRect(r, lcd.pal[cur])
}

// oamline draws up to 10 sprites on the current scanline
func (lcd *display) oamline() {
	// TODO sprite priorities for overlapping sprites at
	// different x-coordinates (lower x-coordinate wins)

	count := 0
	for idx := 0; idx < 0xA0 && count < 10; idx += 4 {
		y := int(lcd.oam[idx]) - 16
		x := int(lcd.oam[idx+1]) - 8
		tile := int(lcd.oam[idx+2])
		info := lcd.oam[idx+3]

		h := 8
		if lcd.spriteSize {
			h = 16
			tile &= 0xFE
		}

		if int(lcd.ly) < y || int(lcd.ly) >= y+h {
			continue
		}
		count++
		if x == -8 || x >= 168 {
			continue
		}

		lcd.spriteLine(tile, x, y, h, info)
	}
}

func (lcd *display) spriteLine(tile, x, y, h int, info byte) {
	masked := info&0x80 == 0x80
	yflip := info&0x40 == 0x40
	xflip := info&0x20 == 0x20
	palidx := (info >> 4) & 1

	tiley := int(lcd.ly) - y
	if yflip {
		tiley = h - 1 - tiley
	}
	tile = tile*16 + tiley*2

	for i := 0; i < 8; i++ {
		xi := byte(x + i)
		if xi >= displayW {
			if xi > 248 { // i.e., xi < 0
				continue
			}
			return
		}
		if masked && lcd.oamLineMask[xi] != 0 {
			continue
		}
		bit := uint(i)
		if !xflip {
			bit = uint(7 - i)
		}
		px := (lcd.vram[tile] >> bit) & 1
		px |= ((lcd.vram[tile+1] >> bit) & 1) << 1
		if px != 0 {
			lcd.lineBuf[xi] = lcd.obp[palidx][px]
		}
	}
}

func (lcd *display) mapAt(map1 bool, x, y int) byte {
	idx := (y/tileH)*mapW + x/tileW
	if map1 {
		idx += 0x1C00
	} else {
		idx += 0x1800
	}
	tile := lcd.vram[idx]
	if lcd.tileData {
		idx = int(tile) * 16
	} else {
		idx = 0x1000 + int(int8(tile))*16
	}
	idx += (y % tileH) * 2
	bit := uint(tileW - 1 - x%tileW)
	px := (lcd.vram[idx] >> bit) & 1
	px |= ((lcd.vram[idx+1] >> bit) & 1) << 1
	return px
}

func calcMode(t int, ly byte) byte {
	if ly >= displayH {
		return modeVBlank
	}
	t %= scanlineTicks
	if t < oamTicks {
		return modeOAM
	}
	if t < oamTicks+vramTicks {
		return modeVRAM
	}
	return modeHBlank
}
<commit_msg>Hide the mouse cursor<commit_after>// Copyright 2011 Kevin Bulusek. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gameboy

import (
	"⚛sdl"
	"time"
)

const (
	modeHBlank = byte(iota)
	modeVBlank
	modeOAM
	modeVRAM
)

const (
	tilesX   = 20
	tilesY   = 18
	tileW    = 8
	tileH    = 8
	mapW     = 32
	mapH     = 32
	displayW = tileW * tilesX
	displayH = tileH * tilesY

	hblankTicks = 204 / 4
	vblankTicks = 4560 / 4
	oamTicks    = 80 / 4
	vramTicks   = 172 / 4

	scanlineTicks = oamTicks + vramTicks + hblankTicks
	refreshTicks  = scanlineTicks*displayH + vblankTicks
)

type display struct {
	*memory
	*sdl.Surface
	pal       [4]uint32
	frameTime int64
	screenW   int
	screenH   int

	clock int

	// LCDC flags
	enable       bool
	windowMap    bool
	windowEnable bool
	tileData     bool
	bgMap        bool
	spriteSize   bool
	spriteEnable bool
	bgEnable     bool

	// STAT flags
	lycInterrupt    bool
	oamInterrupt    bool
	vblankInterrupt bool
	hblankInterrupt bool
	mode            byte

	// LCD registers
	ly       byte
	scy, scx byte
	wy, wx   byte

	bgp [4]byte
	obp [2][4]byte

	// Scanlines are rendered to here first, and then drawn to the
	// display - rather than each layer accessing the display
	// separately (which is slow).
	lineBuf [displayW]byte

	// When rendering a scanline this is zeroed out, then
	// logical-ORed with the pixels from the BG and window. This
	// is then used to lookup which pixels can be painted in
	// sprites that are to be obscured by the background layers.
	oamLineMask [displayW]byte
}

func newDisplay(m *memory) *display {
	sdl.WM_SetCaption(m.rom.title(), "")
	lcd := display{memory: m}
	lcd.screenW = displayW * m.config.Scale
	lcd.screenH = displayH * m.config.Scale
	lcd.Surface = sdl.SetVideoMode(lcd.screenW, lcd.screenH, 0,
		sdl.DOUBLEBUF)
	lcd.pal[0] = sdl.MapRGBA(lcd.Format, 0x9B, 0xBC, 0x0F, 0)
	lcd.pal[1] = sdl.MapRGBA(lcd.Format, 0x8B, 0xAC, 0x0F, 0)
	lcd.pal[2] = sdl.MapRGBA(lcd.Format, 0x30, 0x62, 0x30, 0)
	lcd.pal[3] = sdl.MapRGBA(lcd.Format, 0x0F, 0x38, 0x0F, 0)
	sdl.ShowCursor(sdl.DISABLE)
	lcd.FillRect(nil, lcd.pal[0])
	lcd.Flip()
	lcd.frameTime = time.Nanoseconds()
	return &lcd
}

func (lcd *display) step(t int) {
	lcd.clock += t
	if lcd.clock >= refreshTicks {
		lcd.clock -= refreshTicks
	}

	lcd.ly = byte(lcd.clock / scanlineTicks)
	lcd.hram[portLY-0xFF00] = lcd.ly
	mode := calcMode(lcd.clock, lcd.ly)
	if mode == lcd.mode {
		return
	}
	lcd.mode = mode

	stat := lcd.readPort(portSTAT)&^3 | mode
	irq := lcd.readPort(portIF)

	switch mode {
	case modeOAM:
		if lcd.oamInterrupt {
			lcd.writePort(portIF, irq|0x02)
		}
		if lyc := lcd.readPort(portLYC); lcd.ly-1 == lyc {
			stat |= 0x04
			if lcd.lycInterrupt {
				lcd.writePort(portIF, irq|0x02)
			}
		} else {
			stat &^= 0x04
		}
	case modeVRAM:
		if lcd.enable {
			lcd.scanline()
		}
	case modeHBlank:
		if lcd.hblankInterrupt {
			lcd.writePort(portIF, irq|0x02)
		}
	case modeVBlank:
		if lcd.vblankInterrupt {
			irq |= 0x02
		}
		lcd.writePort(portIF, irq|0x01)
		lcd.Flip()

		// while audio is playing, we let it control the
		// emulation speed
		if !lcd.audio.enable {
			lcd.delay()
		}
	}

	lcd.writePort(portSTAT, stat)
}

func (lcd *display) delay() {
	now := time.Nanoseconds()
	delta := now - lcd.frameTime
	target := 16742706 - delta
	if target > 0 {
		time.Sleep(target)
	}
	lcd.frameTime = time.Nanoseconds()
}

func (lcd *display) scanline() {
	for i := 0; i < displayW; i++ {
		lcd.oamLineMask[i] = 0
		lcd.lineBuf[i] = 0
	}

	if lcd.bgEnable {
		lcd.mapline(lcd.bgMap, byte(0), lcd.scx, lcd.scy)
	}

	if lcd.windowEnable {
		if lcd.wx < 167 && lcd.wy < 144 && lcd.ly >= lcd.wy {
			x := int(lcd.wx) - 7
			xoff := -x
			if x < 0 {
				x = 0
			}
			lcd.mapline(lcd.windowMap, byte(x),
				byte(xoff), byte(-lcd.wy))
		}
	}

	if lcd.spriteEnable {
		lcd.oamline()
	}

	lcd.flushline()
}

func (lcd *display) mapline(map1 bool, x, xoff, yoff byte) {
	y := lcd.ly + yoff
	for ; x < displayW; x++ {
		b := lcd.mapAt(map1, int(x+xoff), int(y))
		lcd.oamLineMask[x] |= b
		lcd.lineBuf[x] = lcd.bgp[b]
	}
}

func (lcd *display) flushline() {
	// Do some simple run-length counting to reduce the number of
	// FillRect calls we need to make.
	scale := uint16(lcd.config.Scale)
	r := &sdl.Rect{0, int16(lcd.ly) * int16(scale), scale, scale}
	cur := lcd.lineBuf[0]
	for x := 1; x < displayW; x++ {
		b := lcd.lineBuf[x]
		if b != cur {
			lcd.FillRect(r, lcd.pal[cur])
			cur = b
			r.X = int16(x) * int16(scale)
			r.W = scale
		} else {
			r.W += scale
		}
	}
	lcd.FillRect(r, lcd.pal[cur])
}

// oamline draws up to 10 sprites on the current scanline
func (lcd *display) oamline() {
	// TODO sprite priorities for overlapping sprites at
	// different x-coordinates (lower x-coordinate wins)

	count := 0
	for idx := 0; idx < 0xA0 && count < 10; idx += 4 {
		y := int(lcd.oam[idx]) - 16
		x := int(lcd.oam[idx+1]) - 8
		tile := int(lcd.oam[idx+2])
		info := lcd.oam[idx+3]

		h := 8
		if lcd.spriteSize {
			h = 16
			tile &= 0xFE
		}

		if int(lcd.ly) < y || int(lcd.ly) >= y+h {
			continue
		}
		count++
		if x == -8 || x >= 168 {
			continue
		}

		lcd.spriteLine(tile, x, y, h, info)
	}
}

func (lcd *display) spriteLine(tile, x, y, h int, info byte) {
	masked := info&0x80 == 0x80
	yflip := info&0x40 == 0x40
	xflip := info&0x20 == 0x20
	palidx := (info >> 4) & 1

	tiley := int(lcd.ly) - y
	if yflip {
		tiley = h - 1 - tiley
	}
	tile = tile*16 + tiley*2

	for i := 0; i < 8; i++ {
		xi := byte(x + i)
		if xi >= displayW {
			if xi > 248 { // i.e., xi < 0
				continue
			}
			return
		}
		if masked && lcd.oamLineMask[xi] != 0 {
			continue
		}
		bit := uint(i)
		if !xflip {
			bit = uint(7 - i)
		}
		px := (lcd.vram[tile] >> bit) & 1
		px |= ((lcd.vram[tile+1] >> bit) & 1) << 1
		if px != 0 {
			lcd.lineBuf[xi] = lcd.obp[palidx][px]
		}
	}
}

func (lcd *display) mapAt(map1 bool, x, y int) byte {
	idx := (y/tileH)*mapW + x/tileW
	if map1 {
		idx += 0x1C00
	} else {
		idx += 0x1800
	}
	tile := lcd.vram[idx]
	if lcd.tileData {
		idx = int(tile) * 16
	} else {
		idx = 0x1000 + int(int8(tile))*16
	}
	idx += (y % tileH) * 2
	bit := uint(tileW - 1 - x%tileW)
	px := (lcd.vram[idx] >> bit) & 1
	px |= ((lcd.vram[idx+1] >> bit) & 1) << 1
	return px
}

func calcMode(t int, ly byte) byte {
	if ly >= displayH {
		return modeVBlank
	}
	t %= scanlineTicks
	if t < oamTicks {
		return modeOAM
	}
	if t < oamTicks+vramTicks {
		return modeVRAM
	}
	return modeHBlank
}
<|endoftext|>


<commit_before>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package log

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"gopkg.in/ini.v1"

	"github.com/go-stack/stack"
	"github.com/inconshreveable/log15"
	"github.com/inconshreveable/log15/term"
)

var Root log15.Logger
var loggersToClose []DisposableHandler

func init() {
	loggersToClose = make([]DisposableHandler, 0)
	Root = log15.Root()
	Root.SetHandler(log15.DiscardHandler())
}

func New(logger string, ctx ...interface{}) Logger {
	params := append([]interface{}{"logger", logger}, ctx...)
	return Root.New(params...)
}

func Trace(format string, v ...interface{}) {
	var message string
	if len(v) > 0 {
		message = fmt.Sprintf(format, v)
	} else {
		message = format
	}

	Root.Debug(message)
}

func Debug(format string, v ...interface{}) {
	var message string
	if len(v) > 0 {
		message = fmt.Sprintf(format, v)
	} else {
		message = format
	}

	Root.Debug(message)
}

func Debug2(message string, v ...interface{}) {
	Root.Debug(message, v...)
}

func Info(format string, v ...interface{}) {
	Root.Info(fmt.Sprintf(format, v))
}

func Info2(message string, v ...interface{}) {
	Root.Info(message, v...)
}

func Warn(format string, v ...interface{}) {
	Root.Warn(fmt.Sprintf(format, v))
}

func Warn2(message string, v ...interface{}) {
	Root.Warn(message, v...)
}

func Error(skip int, format string, v ...interface{}) {
	Root.Error(fmt.Sprintf(format, v))
}

func Error2(message string, v ...interface{}) {
	Root.Error(message, v...)
}

func Critical(skip int, format string, v ...interface{}) {
	Root.Crit(fmt.Sprintf(format, v))
}

func Fatal(skip int, format string, v ...interface{}) {
	Root.Crit(fmt.Sprintf(format, v))
	Close()
	os.Exit(1)
}

func Close() {
	for _, logger := range loggersToClose {
		logger.Close()
	}
	loggersToClose = make([]DisposableHandler, 0)
}

var logLevels = map[string]log15.Lvl{
	"trace":    log15.LvlDebug,
	"debug":    log15.LvlDebug,
	"info":     log15.LvlInfo,
	"warn":     log15.LvlWarn,
	"error":    log15.LvlError,
	"critical": log15.LvlCrit,
}

func getLogLevelFromConfig(key string, defaultName string, cfg *ini.File) (string, log15.Lvl) {
	levelName := cfg.Section(key).Key("level").MustString("info")
	levelName = strings.ToLower(levelName)
	level := getLogLevelFromString(levelName)
	return levelName, level
}

func getLogLevelFromString(levelName string) log15.Lvl {
	level, ok := logLevels[levelName]

	if !ok {
		Root.Error("Unknown log level", "level", levelName)
		return log15.LvlError
	}

	return level
}

func getFilters(filterStrArray []string) map[string]log15.Lvl {
	filterMap := make(map[string]log15.Lvl)

	for _, filterStr := range filterStrArray {
		parts := strings.Split(filterStr, ":")
		if len(parts) > 1 {
			filterMap[parts[0]] = getLogLevelFromString(parts[1])
		}
	}

	return filterMap
}

func getLogFormat(format string) log15.Format {
	switch format {
	case "console":
		if term.IsTty(os.Stdout.Fd()) {
			return log15.TerminalFormat()
		}
		return log15.LogfmtFormat()
	case "text":
		return log15.LogfmtFormat()
	case "json":
		return log15.JsonFormat()
	default:
		return log15.LogfmtFormat()
	}
}

func ReadLoggingConfig(modes []string, logsPath string, cfg *ini.File) {
	Close()

	defaultLevelName, _ := getLogLevelFromConfig("log", "info", cfg)
	defaultFilters := getFilters(cfg.Section("log").Key("filters").Strings(" "))

	handlers := make([]log15.Handler, 0)

	for _, mode := range modes {
		mode = strings.TrimSpace(mode)
		sec, err := cfg.GetSection("log." + mode)
		if err != nil {
			Root.Error("Unknown log mode", "mode", mode)
		}

		// Log level.
		_, level := getLogLevelFromConfig("log."+mode, defaultLevelName, cfg)
		modeFilters := getFilters(sec.Key("filters").Strings(" "))
		format := getLogFormat(sec.Key("format").MustString(""))

		var handler log15.Handler

		// Generate log configuration.
		switch mode {
		case "console":
			handler = log15.StreamHandler(os.Stdout, format)
		case "file":
			fileName := sec.Key("file_name").MustString(filepath.Join(logsPath, "grafana.log"))
			os.MkdirAll(filepath.Dir(fileName), os.ModePerm)
			fileHandler := NewFileWriter()
			fileHandler.Filename = fileName
			fileHandler.Format = format
			fileHandler.Rotate = sec.Key("log_rotate").MustBool(true)
			fileHandler.Maxlines = sec.Key("max_lines").MustInt(1000000)
			fileHandler.Maxsize = 1 << uint(sec.Key("max_size_shift").MustInt(28))
			fileHandler.Daily = sec.Key("daily_rotate").MustBool(true)
			fileHandler.Maxdays = sec.Key("max_days").MustInt64(7)
			fileHandler.Init()

			loggersToClose = append(loggersToClose, fileHandler)
			handler = fileHandler
		case "syslog":
			sysLogHandler := NewSyslog(sec, format)

			loggersToClose = append(loggersToClose, sysLogHandler)
			handler = sysLogHandler
		}

		for key, value := range defaultFilters {
			if _, exist := modeFilters[key]; !exist {
				modeFilters[key] = value
			}
		}

		handler = LogFilterHandler(level, modeFilters, handler)
		handlers = append(handlers, handler)
	}

	Root.SetHandler(log15.MultiHandler(handlers...))
}

func LogFilterHandler(maxLevel log15.Lvl, filters map[string]log15.Lvl, h log15.Handler) log15.Handler {
	return log15.FilterHandler(func(r *log15.Record) (pass bool) {

		if len(filters) > 0 {
			for i := 0; i < len(r.Ctx); i += 2 {
				key := r.Ctx[i].(string)
				if key == "logger" {
					loggerName, strOk := r.Ctx[i+1].(string)
					if strOk {
						if filterLevel, ok := filters[loggerName]; ok {
							return r.Lvl <= filterLevel
						}
					}
				}
			}
		}

		return r.Lvl <= maxLevel
	}, h)
}

func Stack(skip int) string {
	call := stack.Caller(skip)
	s := stack.Trace().TrimBelow(call).TrimRuntime()
	return s.String()
}
<commit_msg>tech(log): fixes extra param logging<commit_after>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package log

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"gopkg.in/ini.v1"

	"github.com/go-stack/stack"
	"github.com/inconshreveable/log15"
	"github.com/inconshreveable/log15/term"
)

var Root log15.Logger
var loggersToClose []DisposableHandler

func init() {
	loggersToClose = make([]DisposableHandler, 0)
	Root = log15.Root()
	Root.SetHandler(log15.DiscardHandler())
}

func New(logger string, ctx ...interface{}) Logger {
	params := append([]interface{}{"logger", logger}, ctx...)
	return Root.New(params...)
}

func Trace(format string, v ...interface{}) {
	var message string
	if len(v) > 0 {
		message = fmt.Sprintf(format, v)
	} else {
		message = format
	}

	Root.Debug(message)
}

func Debug(format string, v ...interface{}) {
	var message string
	if len(v) > 0 {
		message = fmt.Sprintf(format, v)
	} else {
		message = format
	}

	Root.Debug(message)
}

func Debug2(message string, v ...interface{}) {
	Root.Debug(message, v...)
}

func Info(format string, v ...interface{}) {
	var message string
	if len(v) > 0 {
		message = fmt.Sprintf(format, v)
	} else {
		message = format
	}

	Root.Info(message)
}

func Info2(message string, v ...interface{}) {
	Root.Info(message, v...)
}

func Warn(format string, v ...interface{}) {
	var message string
	if len(v) > 0 {
		message = fmt.Sprintf(format, v)
	} else {
		message = format
	}

	Root.Warn(message)
}

func Warn2(message string, v ...interface{}) {
	Root.Warn(message, v...)
}

func Error(skip int, format string, v ...interface{}) {
	Root.Error(fmt.Sprintf(format, v))
}

func Error2(message string, v ...interface{}) {
	Root.Error(message, v...)
}

func Critical(skip int, format string, v ...interface{}) {
	Root.Crit(fmt.Sprintf(format, v))
}

func Fatal(skip int, format string, v ...interface{}) {
	Root.Crit(fmt.Sprintf(format, v))
	Close()
	os.Exit(1)
}

func Close() {
	for _, logger := range loggersToClose {
		logger.Close()
	}
	loggersToClose = make([]DisposableHandler, 0)
}

var logLevels = map[string]log15.Lvl{
	"trace":    log15.LvlDebug,
	"debug":    log15.LvlDebug,
	"info":     log15.LvlInfo,
	"warn":     log15.LvlWarn,
	"error":    log15.LvlError,
	"critical": log15.LvlCrit,
}

func getLogLevelFromConfig(key string, defaultName string, cfg *ini.File) (string, log15.Lvl) {
	levelName := cfg.Section(key).Key("level").MustString("info")
	levelName = strings.ToLower(levelName)
	level := getLogLevelFromString(levelName)
	return levelName, level
}

func getLogLevelFromString(levelName string) log15.Lvl {
	level, ok := logLevels[levelName]

	if !ok {
		Root.Error("Unknown log level", "level", levelName)
		return log15.LvlError
	}

	return level
}

func getFilters(filterStrArray []string) map[string]log15.Lvl {
	filterMap := make(map[string]log15.Lvl)

	for _, filterStr := range filterStrArray {
		parts := strings.Split(filterStr, ":")
		if len(parts) > 1 {
			filterMap[parts[0]] = getLogLevelFromString(parts[1])
		}
	}

	return filterMap
}

func getLogFormat(format string) log15.Format {
	switch format {
	case "console":
		if term.IsTty(os.Stdout.Fd()) {
			return log15.TerminalFormat()
		}
		return log15.LogfmtFormat()
	case "text":
		return log15.LogfmtFormat()
	case "json":
		return log15.JsonFormat()
	default:
		return log15.LogfmtFormat()
	}
}

func ReadLoggingConfig(modes []string, logsPath string, cfg *ini.File) {
	Close()

	defaultLevelName, _ := getLogLevelFromConfig("log", "info", cfg)
	defaultFilters := getFilters(cfg.Section("log").Key("filters").Strings(" "))

	handlers := make([]log15.Handler, 0)

	for _, mode := range modes {
		mode = strings.TrimSpace(mode)
		sec, err := cfg.GetSection("log." + mode)
		if err != nil {
			Root.Error("Unknown log mode", "mode", mode)
		}

		// Log level.
		_, level := getLogLevelFromConfig("log."+mode, defaultLevelName, cfg)
		modeFilters := getFilters(sec.Key("filters").Strings(" "))
		format := getLogFormat(sec.Key("format").MustString(""))

		var handler log15.Handler

		// Generate log configuration.
		switch mode {
		case "console":
			handler = log15.StreamHandler(os.Stdout, format)
		case "file":
			fileName := sec.Key("file_name").MustString(filepath.Join(logsPath, "grafana.log"))
			os.MkdirAll(filepath.Dir(fileName), os.ModePerm)
			fileHandler := NewFileWriter()
			fileHandler.Filename = fileName
			fileHandler.Format = format
			fileHandler.Rotate = sec.Key("log_rotate").MustBool(true)
			fileHandler.Maxlines = sec.Key("max_lines").MustInt(1000000)
			fileHandler.Maxsize = 1 << uint(sec.Key("max_size_shift").MustInt(28))
			fileHandler.Daily = sec.Key("daily_rotate").MustBool(true)
			fileHandler.Maxdays = sec.Key("max_days").MustInt64(7)
			fileHandler.Init()

			loggersToClose = append(loggersToClose, fileHandler)
			handler = fileHandler
		case "syslog":
			sysLogHandler := NewSyslog(sec, format)

			loggersToClose = append(loggersToClose, sysLogHandler)
			handler = sysLogHandler
		}

		for key, value := range defaultFilters {
			if _, exist := modeFilters[key]; !exist {
				modeFilters[key] = value
			}
		}

		handler = LogFilterHandler(level, modeFilters, handler)
		handlers = append(handlers, handler)
	}

	Root.SetHandler(log15.MultiHandler(handlers...))
}

func LogFilterHandler(maxLevel log15.Lvl, filters map[string]log15.Lvl, h log15.Handler) log15.Handler {
	return log15.FilterHandler(func(r *log15.Record) (pass bool) {

		if len(filters) > 0 {
			for i := 0; i < len(r.Ctx); i += 2 {
				key := r.Ctx[i].(string)
				if key == "logger" {
					loggerName, strOk := r.Ctx[i+1].(string)
					if strOk {
						if filterLevel, ok := filters[loggerName]; ok {
							return r.Lvl <= filterLevel
						}
					}
				}
			}
		}

		return r.Lvl <= maxLevel
	}, h)
}

func Stack(skip int) string {
	call := stack.Caller(skip)
	s := stack.Trace().TrimBelow(call).TrimRuntime()
	return s.String()
}
<|endoftext|>


<commit_before>package place

import (
	"bytes"
	"crypto/sha256"
	"encoding/gob"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/util"
	"gopkg.in/h2non/filetype.v1"
	"gopkg.in/h2non/filetype.v1/types"
)

type Place struct {
	ConfigPath string
	Config     Config
	Db         *leveldb.DB
	Preview    bool
	Number     int
	History    History
	TmpFile    map[string][]byte
}

func (p *Place) Run(files []string) {
	p.loadConfig()
	p.getNumber()
	p.History.Files = make(map[string]string)
	log.Debug("Run:", files)
	for _, v := range files {
		p.run(v)
	}
	if !p.Preview {
		p.saveHistory()
	}
}

func (p *Place) getNumber() {
	name := []byte("count")
	bs, _ := p.Db.Get(name, nil)
	p.Number = Bytes2Int(bs)
	p.Number += 1
	log.Debugf("操作次数: %d", p.Number)
	p.Db.Put(name, Int2Bytes(p.Number), nil)
}

func (p *Place) saveHistory() {
	log.Debugf("保存历史: %v", p.History.Files)
	p.History.Timestamp = time.Now()
	bf := bytes.NewBuffer(nil)
	enc := gob.NewEncoder(bf)
	err := enc.Encode(p.History)
	if err != nil {
		panic("操作记录编码失败")
	}
	p.Db.Put([]byte(fmt.Sprintf("no-%04d", p.Number)), bf.Bytes(), nil)
}

func (p *Place) loadConfig() {
	p.Config = Config{}
	p.Config.Load(p.ConfigPath)
}

func (p *Place) run(file string) {
	info, err := os.Stat(file)
	if os.IsNotExist(err) {
		log.Error("文件未找到: ", file)
		return
	}
	log.Debugf("%s 目录: %t", file, info.IsDir())
	if info.IsDir() {
		log.Info("处理目录: ", file)
	} else {
		log.Debug("处理文件: ", file)
		bs, _ := ioutil.ReadFile(file)
		head := bs[:261]
		if filetype.IsImage(head) {
			log.Debugf("文件 %s 是图片", file)
		} else {
			log.Debugf("文件 %s 不是图片", file)
		}
		hash := sha256.New()
		hash.Write(bs)
		log.Debugf("%s sha256: %x", file, hash.Sum(nil))
		kind, _ := filetype.Match(head)
		log.Debugf("扩展文件名: %s type: %s", path.Ext(file), kind)
		old := p.find(hash.Sum(nil))
		if old == "" {
			newFile, err := p.moveName(kind, file, info)
			if err == nil {
				if p.Preview {
					log.Infof("预览: %s >>> %s", file, newFile)
				} else {
					log.Infof("移动: %s >>> %s", file, newFile)
					p.History.Files[file] = newFile
					os.Rename(file, newFile)
					p.Db.Put(BytesPrefix("f-", hash.Sum(nil)), []byte(newFile), nil)
				}
			} else {
				log.Errorf("%s: %s", file, err)
			}
		} else {
			log.Warnf("文件 %s 重复, 原文件 %s", file, old)
		}
	}
}

func (p *Place) find(hash []byte) string {
	file, err := p.Db.Get(BytesPrefix("f-", hash), nil)
	if err != nil {
		return ""
	}
	return string(file)
}

func (p *Place) moveName(t types.Type, file string, info os.FileInfo) (string, error) {
	log.Debugf("mime: %s, subtype: %s, 搬移文件: %s", t.MIME.Type, t.MIME.Subtype, file)
	ext := path.Ext(file)
	for _, ap := range p.Config.Paths {
		if (ap.Mime == t.MIME.Type && ap.Subtype == t.MIME.Subtype) ||
			(ap.Mime == "" && ap.Subtype == "" && t == filetype.Unknown) ||
			(ap.Mime == t.MIME.Type && ap.Subtype == "") {
			if ap.Ext == "" || ap.Ext == ext {
				newDir := ap.Dir
				if ap.Subdir != "" {
					newDir = path.Join(newDir, TimeFormat(info.ModTime(), ap.Subdir))
				}
				dir := ToPath(newDir)
				newFile := path.Join(dir, path.Base(file))
				info, err := os.Stat(newFile)
				if os.IsNotExist(err) {
					return newFile, nil
				}
				if info.IsDir() {
					return file, errors.New("同名目录已经存在")
				} else {
					return file, errors.New("同名文件已经存在")
				}
			}
		}
	}
	return file, errors.New("无匹配设置")
}

func (p *Place) Scan() {
	log.Info("开始目录扫描")
	p.loadConfig()

	p.TmpFile = make(map[string][]byte)
	iter := p.Db.NewIterator(util.BytesPrefix([]byte("f-")), nil)
	for iter.Next() {
		file := string(iter.Value())
		p.TmpFile[file] = iter.Key()
		log.Debug("缓存文件: ", file)
	}
	iter.Release()

	for _, ap := range p.Config.Paths {
		log.Debug("扫描目录: ", ap.Dir)
		p.scanning(ToPath(ap.Dir))
	}

	for file, key := range p.TmpFile {
		log.Debug("删除缓存: ", file)
		p.Db.Delete(key, nil)
	}
}

func (p *Place) scanning(dir string) {
	err := filepath.Walk(dir, func(filename string, fi os.FileInfo, err error) error {
		if filename == dir {
			return nil
		}
		if fi.IsDir() {
			p.scanning(filename)
		} else {
			log.Debug("扫描文件: ", filename)
			_, ok := p.TmpFile[filename]
			if ok {
				delete(p.TmpFile, filename)
			} else {
				bs, _ := ioutil.ReadFile(filename)
				hash := sha256.New()
				hash.Write(bs)
				key := BytesPrefix("f-", hash.Sum(nil))
				old, err := p.Db.Get(key, nil)
				if err == nil {
					if filename != string(old) {
						log.Warnf("文件重复: %s = %s", filename, old)
					}
				} else {
					log.Info("新增文件: ", filename)
					p.Db.Put(key, []byte(filename), nil)
				}
			}
		}
		return nil
	})
	if err != nil {
		log.Error("目录扫描错误: ", err)
	}
}

func (p *Place) Reset() {
	log.Info("开始根据配置重置目录")

	p.TmpFile = make(map[string][]byte)
	iter := p.Db.NewIterator(util.BytesPrefix([]byte("f-")), nil)
	for iter.Next() {
		file := string(iter.Value())
		p.TmpFile[file] = iter.Key()
		log.Debug("缓存文件: ", file)
	}
	iter.Release()

	for _, ap := range p.Config.Paths {
		log.Debug("扫描目录: ", ap.Dir)
		p.reset(ToPath(ap.Dir))
	}
}

func (p *Place) reset(dir string) {
	count := 0
	err := filepath.Walk(dir, func(filename string, fi os.FileInfo, err error) error {
		if filename == dir {
			return nil
		}
		count += 1
		if fi.IsDir() {
			p.reset(filename)
		} else {
			log.Debug("重置文件: ", filename)
			bs, _ := ioutil.ReadFile(filename)
			head := bs[:261]
			kind, _ := filetype.Match(head)
			newFile, err := p.moveName(kind, filename, fi)
			if err != nil || newFile == filename {
				log.Debug("无需移动: ", filename)
			} else {
				log.Infof("移动: %s >>> %s", filename, newFile)
				os.Rename(filename, newFile)
				key, _ := p.TmpFile[filename]
				p.Db.Put(key, []byte(newFile), nil)
			}
		}
		return nil
	})
	if err == nil {
		log.Debugf("%s 子文件数量: %d", dir, count)
		if count == 0 {
			log.Info("删除空目录: ", dir)
			os.Remove(dir)
		}
	} else {
		log.Error("目录扫描错误: ", err)
	}
}
<commit_msg>fix #10<commit_after>package place

import (
	"bytes"
	"crypto/sha256"
	"encoding/gob"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/util"
	"gopkg.in/h2non/filetype.v1"
	"gopkg.in/h2non/filetype.v1/types"
)

type Place struct {
	ConfigPath string
	Config     Config
	Db         *leveldb.DB
	Preview    bool
	Number     int
	History    History
	TmpFile    map[string][]byte
}

func (p *Place) Run(files []string) {
	p.loadConfig()
	p.getNumber()
	p.History.Files = make(map[string]string)
	log.Debug("Run:", files)
	for _, v := range files {
		p.run(v)
	}
	if !p.Preview {
		p.saveHistory()
	}
}

func (p *Place) getNumber() {
	name := []byte("count")
	bs, _ := p.Db.Get(name, nil)
	p.Number = Bytes2Int(bs)
	p.Number += 1
	log.Debugf("操作次数: %d", p.Number)
	p.Db.Put(name, Int2Bytes(p.Number), nil)
}

func (p *Place) saveHistory() {
	log.Debugf("保存历史: %v", p.History.Files)
	p.History.Timestamp = time.Now()
	bf := bytes.NewBuffer(nil)
	enc := gob.NewEncoder(bf)
	err := enc.Encode(p.History)
	if err != nil {
		panic("操作记录编码失败")
	}
	p.Db.Put([]byte(fmt.Sprintf("no-%04d", p.Number)), bf.Bytes(), nil)
}

func (p *Place) loadConfig() {
	p.Config = Config{}
	p.Config.Load(p.ConfigPath)
}

func (p *Place) run(file string) {
	info, err := os.Stat(file)
	if os.IsNotExist(err) {
		log.Error("文件未找到: ", file)
		return
	}
	log.Debugf("%s 目录: %t", file, info.IsDir())
	if info.IsDir() {
		log.Info("处理目录: ", file)
	} else {
		log.Debug("处理文件: ", file)
		bs, _ := ioutil.ReadFile(file)
		head := bs[:261]
		if filetype.IsImage(head) {
			log.Debugf("文件 %s 是图片", file)
		} else {
			log.Debugf("文件 %s 不是图片", file)
		}
		hash := sha256.New()
		hash.Write(bs)
		log.Debugf("%s sha256: %x", file, hash.Sum(nil))
		kind, _ := filetype.Match(head)
		log.Debugf("扩展文件名: %s type: %s", path.Ext(file), kind)
		old := p.find(hash.Sum(nil))
		if old == "" {
			newFile, err := p.moveName(kind, file, info)
			if err == nil {
				if p.Preview {
					log.Infof("预览: %s >>> %s", file, newFile)
				} else {
					log.Infof("移动: %s >>> %s", file, newFile)
					p.History.Files[file] = newFile
					os.Rename(file, newFile)
					p.Db.Put(BytesPrefix("f-", hash.Sum(nil)), []byte(newFile), nil)
				}
			} else {
				log.Errorf("%s: %s", file, err)
			}
		} else {
			log.Warnf("文件 %s 重复, 原文件 %s", file, old)
		}
	}
}

func (p *Place) find(hash []byte) string {
	file, err := p.Db.Get(BytesPrefix("f-", hash), nil)
	if err != nil {
		return ""
	}
	return string(file)
}

func (p *Place) moveName(t types.Type, file string, info os.FileInfo) (string, error) {
	log.Debugf("mime: %s, subtype: %s, 搬移文件: %s", t.MIME.Type, t.MIME.Subtype, file)
	ext := path.Ext(file)
	for _, ap := range p.Config.Paths {
		if (strings.EqualFold(ap.Mime, t.MIME.Type) && strings.EqualFold(ap.Subtype, t.MIME.Subtype)) ||
			(ap.Mime == "" && ap.Subtype == "" && t == filetype.Unknown) ||
			(strings.EqualFold(ap.Mime, t.MIME.Type) && ap.Subtype == "") {
			if ap.Ext == "" || strings.EqualFold(ap.Ext, ext) {
				newDir := ap.Dir
				if ap.Subdir != "" {
					newDir = path.Join(newDir, TimeFormat(info.ModTime(), ap.Subdir))
				}
				dir := ToPath(newDir)
				newFile := path.Join(dir, path.Base(file))
				info, err := os.Stat(newFile)
				if os.IsNotExist(err) {
					return newFile, nil
				}
				if info.IsDir() {
					return file, errors.New("同名目录已经存在")
				} else {
					return file, errors.New("同名文件已经存在")
				}
			}
		}
	}
	return file, errors.New("无匹配设置")
}

func (p *Place) Scan() {
	log.Info("开始目录扫描")
	p.loadConfig()

	p.TmpFile = make(map[string][]byte)
	iter := p.Db.NewIterator(util.BytesPrefix([]byte("f-")), nil)
	for iter.Next() {
		file := string(iter.Value())
		p.TmpFile[file] = iter.Key()
		log.Debug("缓存文件: ", file)
	}
	iter.Release()

	for _, ap := range p.Config.Paths {
		log.Debug("扫描目录: ", ap.Dir)
		p.scanning(ToPath(ap.Dir))
	}

	for file, key := range p.TmpFile {
		log.Debug("删除缓存: ", file)
		p.Db.Delete(key, nil)
	}
}

func (p *Place) scanning(dir string) {
	err := filepath.Walk(dir, func(filename string, fi os.FileInfo, err error) error {
		if filename == dir {
			return nil
		}
		if fi.IsDir() {
			p.scanning(filename)
		} else {
			log.Debug("扫描文件: ", filename)
			_, ok := p.TmpFile[filename]
			if ok {
				delete(p.TmpFile, filename)
			} else {
				bs, _ := ioutil.ReadFile(filename)
				hash := sha256.New()
				hash.Write(bs)
				key := BytesPrefix("f-", hash.Sum(nil))
				old, err := p.Db.Get(key, nil)
				if err == nil {
					if filename != string(old) {
						log.Warnf("文件重复: %s = %s", filename, old)
					}
				} else {
					log.Info("新增文件: ", filename)
					p.Db.Put(key, []byte(filename), nil)
				}
			}
		}
		return nil
	})
	if err != nil {
		log.Error("目录扫描错误: ", err)
	}
}

func (p *Place) Reset() {
	log.Info("开始根据配置重置目录")

	p.TmpFile = make(map[string][]byte)
	iter := p.Db.NewIterator(util.BytesPrefix([]byte("f-")), nil)
	for iter.Next() {
		file := string(iter.Value())
		p.TmpFile[file] = iter.Key()
		log.Debug("缓存文件: ", file)
	}
	iter.Release()

	for _, ap := range p.Config.Paths {
		log.Debug("扫描目录: ", ap.Dir)
		p.reset(ToPath(ap.Dir))
	}
}

func (p *Place) reset(dir string) {
	count := 0
	err := filepath.Walk(dir, func(filename string, fi os.FileInfo, err error) error {
		if filename == dir {
			return nil
		}
		count += 1
		if fi.IsDir() {
			p.reset(filename)
		} else {
			log.Debug("重置文件: ", filename)
			bs, _ := ioutil.ReadFile(filename)
			head := bs[:261]
			kind, _ := filetype.Match(head)
			newFile, err := p.moveName(kind, filename, fi)
			if err != nil || newFile == filename {
				log.Debug("无需移动: ", filename)
			} else {
				log.Infof("移动: %s >>> %s", filename, newFile)
				os.Rename(filename, newFile)
				key, _ := p.TmpFile[filename]
				p.Db.Put(key, []byte(newFile), nil)
			}
		}
		return nil
	})
	if err == nil {
		log.Debugf("%s 子文件数量: %d", dir, count)
		if count == 0 {
			log.Info("删除空目录: ", dir)
			os.Remove(dir)
		}
	} else {
		log.Error("目录扫描错误: ", err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package files

import (
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/coreos/ignition/v2/config/v3_2_experimental/types"
	"github.com/coreos/ignition/v2/internal/exec/util"
	"github.com/coreos/ignition/v2/internal/log"
)

// createFilesystemsEntries creates the files described in config.Storage.{Files,Directories}.
func (s *stage) createFilesystemsEntries(config types.Config) error {
	s.Logger.PushPrefix("createFilesystemsFiles")
	defer s.Logger.PopPrefix()

	entries, err := s.getOrderedCreationList(config)
	if err != nil {
		return err
	}

	if err := s.createEntries(entries); err != nil {
		return fmt.Errorf("failed to create files: %v", err)
	}

	return nil
}

// filesystemEntry represent a thing that knows how to create itself.
type filesystemEntry interface {
	// create creates the entry if specified. It assumes that if overwrite=true then any existing
	// files at the path will have been deleted.
	create(l *log.Logger, u util.Util) error
	node() types.Node
}

type fileEntry types.File

func (tmp fileEntry) node() types.Node {
	return types.File(tmp).Node
}

func (tmp fileEntry) create(l *log.Logger, u util.Util) error {
	f := types.File(tmp)

	empty := "" // golang--

	st, err := os.Lstat(f.Path)
	regular := (st == nil) || st.Mode().IsRegular()
	switch {
	case os.IsNotExist(err) && f.Contents.Source == nil:
		// set f.Contents so we create an empty file
		f.Contents.Source = &empty
	case os.IsNotExist(err):
		break
	case err != nil:
		return err
	// Cases where there is file there
	case !regular:
		return fmt.Errorf("error creating file %q: A non regular file exists there already and overwrite is false", f.Path)
	case f.Contents.Source != nil:
		return fmt.Errorf("error creating file %q: A file exists there already and overwrite is false", f.Path)
	case regular && f.Contents.Source == nil:
		break
	default:
		return fmt.Errorf("Ignition encountered an internal error processing %q and must die now. Please file a bug", f.Path)
	}

	fetchOps, err := u.PrepareFetches(l, f)
	if err != nil {
		return fmt.Errorf("failed to resolve file %q: %v", f.Path, err)
	}

	for _, op := range fetchOps {
		msg := "writing file %q"
		if op.Append {
			msg = "appending to file %q"
		}
		if err := l.LogOp(
			func() error {
				return u.PerformFetch(op)
			}, msg, f.Path,
		); err != nil {
			return fmt.Errorf("failed to create file %q: %v", op.Node.Path, err)
		}
	}
	if err := u.SetPermissions(f.Mode, f.Node); err != nil {
		return fmt.Errorf("error setting file permissions for %s: %v", f.Path, err)
	}
	return nil
}

type dirEntry types.Directory

func (tmp dirEntry) node() types.Node {
	return types.Directory(tmp).Node
}

func (tmp dirEntry) create(l *log.Logger, u util.Util) error {
	d := types.Directory(tmp)
	st, err := os.Lstat(d.Path)
	switch {
	case os.IsNotExist(err):
		// use default perms, we'll fix it later
		if err := os.MkdirAll(d.Path, util.DefaultDirectoryPermissions); err != nil {
			return fmt.Errorf("Failed to create directory %s: %v", d.Path, err)
		}
	case err != nil:
		return fmt.Errorf("stat() failed on %s: %v", d.Path, err)
	case !st.Mode().IsDir():
		return fmt.Errorf("error creating directory %s: A non-directory already exists and overwrite is false", d.Path)
	}

	if err := u.SetPermissions(d.Mode, d.Node); err != nil {
		return fmt.Errorf("error setting directory permissions for %s: %v", d.Path, err)
	}
	return nil
}

type linkEntry types.Link

func (tmp linkEntry) node() types.Node {
	return types.Link(tmp).Node
}

func (tmp linkEntry) create(l *log.Logger, u util.Util) error {
	s := types.Link(tmp)
	hard := s.Hard != nil && *s.Hard
	st, err := os.Lstat(s.Path)
	switch {
	case os.IsNotExist(err):
		break
	case err != nil:
		return fmt.Errorf("stat() failed on %s: %v", s.Path, err)
	case hard:
		// check that the file at that path points to the same inode as target
		targetPath, err := u.JoinPath(s.Target)
		if err != nil {
			return fmt.Errorf("error resolving target path of hard link %s: %v", s.Path, err)
		}
		targetst, err := os.Lstat(targetPath)
		if err != nil {
			return fmt.Errorf("error creating hard link %s: target does not exist or stat() returned an err: %v", s.Path, err)
		}
		if !os.SameFile(st, targetst) {
			return fmt.Errorf("error creating hard link %s: a file already exists at that path but is not the target and overwrite is false", s.Path)
		}
		l.Info("Hardlink %s to %s already exists, doing nothing", s.Path, s.Target)
		return nil
	case !hard:
		// if the existing file is a symlink, check that its target is correct
		if st.Mode()&os.ModeSymlink != 0 {
			if target, err := os.Readlink(s.Path); err != nil {
				return fmt.Errorf("error reading link at %s: %v", s.Path, err)
			} else if filepath.Clean(target) != filepath.Clean(s.Target) {
				return fmt.Errorf("error creating symlink %s: a symlink exists at that path but points to %s, not %s and overwrite is false", s.Path, target, s.Target)
			} else {
				l.Info("Symlink %s to %s already exists, doing nothing", s.Path, s.Target)
				return nil
			}
		}
		return fmt.Errorf("error creating symlink %s: a non-symlink already exists at that path and overwrite is false", s.Path)
	}

	if err := l.LogOp(
		func() error {
			return u.WriteLink(s)
		}, "writing link %q -> %q", s.Path, s.Target,
	); err != nil {
		return fmt.Errorf("failed to create link %q: %v", s.Path, err)
	}

	return nil
}

// getOrderedCreationList resolves all symlinks in the node paths and sets the path to be
// prepended by the sysroot. It orders the list from shallowest (e.g. /a) to deepeset
// (e.g. /a/b/c/d/e).
func (s stage) getOrderedCreationList(config types.Config) ([]filesystemEntry, error) {
	entries := []filesystemEntry{}
	// Map from paths in the config to where they resolve for duplicate checking
	paths := map[string]string{}
	for _, d := range config.Storage.Directories {
		path, err := s.JoinPath(d.Path)
		if err != nil {
			return nil, err
		}
		if existing, ok := paths[path]; ok {
			return nil, fmt.Errorf("Directory at %s resolved to %s after symlink chasing, but another entry with path %s also resolves there",
				d.Path, path, existing)
		}
		paths[path] = d.Path
		d.Path = path
		entries = append(entries, dirEntry(d))
	}

	for _, f := range config.Storage.Files {
		path, err := s.JoinPath(f.Path)
		if err != nil {
			return nil, err
		}
		if existing, ok := paths[path]; ok {
			return nil, fmt.Errorf("File at %s resolved to %s after symlink chasing, but another entry with path %s also resolves there",
				f.Path, path, existing)
		}
		paths[path] = f.Path
		f.Path = path
		entries = append(entries, fileEntry(f))
	}

	for _, l := range config.Storage.Links {
		path, err := s.JoinPath(l.Path)
		if err != nil {
			return nil, err
		}
		if existing, ok := paths[path]; ok {
			return nil, fmt.Errorf("Link at %s resolved to %s after symlink chasing, but another entry with path %s also resolves there",
				l.Path, path, existing)
		}
		paths[path] = l.Path
		l.Path = path
		entries = append(entries, linkEntry(l))
	}
	sort.Slice(entries, func(i, j int) bool { return util.Depth(entries[i].node().Path) < util.Depth(entries[j].node().Path) })

	return entries, nil
}

func (s *stage) removePathOnOverwrite(e filesystemEntry) error {
	if e.node().Overwrite != nil && *e.node().Overwrite {
		return os.RemoveAll(e.node().Path)
	}
	return nil
}

func (s *stage) relabelDirsForFile(path string) error {
	if !s.relabeling() {
		return nil
	}

	missing_dir, err := util.FindFirstMissingDirForFile(path)
	if err != nil {
		return err
	}

	// trim off prefix since this needs to be relative to the sysroot
	s.relabel(missing_dir[len(s.DestDir):])
	return nil
}

// createEntries creates any files or directories listed for the filesystem in Storage.{Files,Directories}.
func (s *stage) createEntries(entries []filesystemEntry) error {
	s.Logger.PushPrefix("createFiles")
	defer s.Logger.PopPrefix()

	for _, e := range entries {
		path := e.node().Path
		if !strings.HasPrefix(path, s.DestDir) {
			panic(fmt.Sprintf("Entry path %s isn't under prefix %s", path, s.DestDir))
		}

		if err := s.relabelDirsForFile(path); err != nil {
			return fmt.Errorf("error relabeling paths for %s: %v", path, err)
		}
		if err := s.removePathOnOverwrite(e); err != nil {
			return fmt.Errorf("error removing existing file %s: %v", path, err)
		}
		if err := e.create(s.Logger, s.Util); err != nil {
			return fmt.Errorf("error creating %s: %v", path, err)
		}
	}
	return nil
}
<commit_msg>stages/files: fix hard links to files that are deeper than the hard link<commit_after>// Copyright 2018 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package files

import (
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/coreos/ignition/v2/config/v3_2_experimental/types"
	"github.com/coreos/ignition/v2/internal/exec/util"
	"github.com/coreos/ignition/v2/internal/log"
)

// createFilesystemsEntries creates the files described in config.Storage.{Files,Directories}.
func (s *stage) createFilesystemsEntries(config types.Config) error {
	s.Logger.PushPrefix("createFilesystemsFiles")
	defer s.Logger.PopPrefix()

	entries, err := s.getOrderedCreationList(config)
	if err != nil {
		return err
	}

	if err := s.createEntries(entries); err != nil {
		return fmt.Errorf("failed to create files: %v", err)
	}

	return nil
}

// filesystemEntry represent a thing that knows how to create itself.
type filesystemEntry interface {
	// create creates the entry if specified. It assumes that if overwrite=true then any existing
	// files at the path will have been deleted.
	create(l *log.Logger, u util.Util) error
	node() types.Node
}

type fileEntry types.File

func (tmp fileEntry) node() types.Node {
	return types.File(tmp).Node
}

func (tmp fileEntry) create(l *log.Logger, u util.Util) error {
	f := types.File(tmp)

	empty := "" // golang--

	st, err := os.Lstat(f.Path)
	regular := (st == nil) || st.Mode().IsRegular()
	switch {
	case os.IsNotExist(err) && f.Contents.Source == nil:
		// set f.Contents so we create an empty file
		f.Contents.Source = &empty
	case os.IsNotExist(err):
		break
	case err != nil:
		return err
	// Cases where there is file there
	case !regular:
		return fmt.Errorf("error creating file %q: A non regular file exists there already and overwrite is false", f.Path)
	case f.Contents.Source != nil:
		return fmt.Errorf("error creating file %q: A file exists there already and overwrite is false", f.Path)
	case regular && f.Contents.Source == nil:
		break
	default:
		return fmt.Errorf("Ignition encountered an internal error processing %q and must die now. Please file a bug", f.Path)
	}

	fetchOps, err := u.PrepareFetches(l, f)
	if err != nil {
		return fmt.Errorf("failed to resolve file %q: %v", f.Path, err)
	}

	for _, op := range fetchOps {
		msg := "writing file %q"
		if op.Append {
			msg = "appending to file %q"
		}
		if err := l.LogOp(
			func() error {
				return u.PerformFetch(op)
			}, msg, f.Path,
		); err != nil {
			return fmt.Errorf("failed to create file %q: %v", op.Node.Path, err)
		}
	}
	if err := u.SetPermissions(f.Mode, f.Node); err != nil {
		return fmt.Errorf("error setting file permissions for %s: %v", f.Path, err)
	}
	return nil
}

type dirEntry types.Directory

func (tmp dirEntry) node() types.Node {
	return types.Directory(tmp).Node
}

func (tmp dirEntry) create(l *log.Logger, u util.Util) error {
	d := types.Directory(tmp)
	st, err := os.Lstat(d.Path)
	switch {
	case os.IsNotExist(err):
		// use default perms, we'll fix it later
		if err := os.MkdirAll(d.Path, util.DefaultDirectoryPermissions); err != nil {
			return fmt.Errorf("Failed to create directory %s: %v", d.Path, err)
		}
	case err != nil:
		return fmt.Errorf("stat() failed on %s: %v", d.Path, err)
	case !st.Mode().IsDir():
		return fmt.Errorf("error creating directory %s: A non-directory already exists and overwrite is false", d.Path)
	}

	if err := u.SetPermissions(d.Mode, d.Node); err != nil {
		return fmt.Errorf("error setting directory permissions for %s: %v", d.Path, err)
	}
	return nil
}

type linkEntry types.Link

func (tmp linkEntry) node() types.Node {
	return types.Link(tmp).Node
}

func (tmp linkEntry) create(l *log.Logger, u util.Util) error {
	s := types.Link(tmp)
	hard := s.Hard != nil && *s.Hard
	st, err := os.Lstat(s.Path)
	switch {
	case os.IsNotExist(err):
		break
	case err != nil:
		return fmt.Errorf("stat() failed on %s: %v", s.Path, err)
	case hard:
		// check that the file at that path points to the same inode as target
		targetPath, err := u.JoinPath(s.Target)
		if err != nil {
			return fmt.Errorf("error resolving target path of hard link %s: %v", s.Path, err)
		}
		targetst, err := os.Lstat(targetPath)
		if err != nil {
			return fmt.Errorf("error creating hard link %s: target does not exist or stat() returned an err: %v", s.Path, err)
		}
		if !os.SameFile(st, targetst) {
			return fmt.Errorf("error creating hard link %s: a file already exists at that path but is not the target and overwrite is false", s.Path)
		}
		l.Info("Hardlink %s to %s already exists, doing nothing", s.Path, s.Target)
		return nil
	case !hard:
		// if the existing file is a symlink, check that its target is correct
		if st.Mode()&os.ModeSymlink != 0 {
			if target, err := os.Readlink(s.Path); err != nil {
				return fmt.Errorf("error reading link at %s: %v", s.Path, err)
			} else if filepath.Clean(target) != filepath.Clean(s.Target) {
				return fmt.Errorf("error creating symlink %s: a symlink exists at that path but points to %s, not %s and overwrite is false", s.Path, target, s.Target)
			} else {
				l.Info("Symlink %s to %s already exists, doing nothing", s.Path, s.Target)
				return nil
			}
		}
		return fmt.Errorf("error creating symlink %s: a non-symlink already exists at that path and overwrite is false", s.Path)
	}

	if err := l.LogOp(
		func() error {
			return u.WriteLink(s)
		}, "writing link %q -> %q", s.Path, s.Target,
	); err != nil {
		return fmt.Errorf("failed to create link %q: %v", s.Path, err)
	}

	return nil
}

// getOrderedCreationList resolves all symlinks in the node paths and sets the path to be
// prepended by the sysroot. It orders the list from shallowest (e.g. /a) to deepeset
// (e.g. /a/b/c/d/e).
func (s stage) getOrderedCreationList(config types.Config) ([]filesystemEntry, error) {
	entries := []filesystemEntry{}
	// Map from paths in the config to where they resolve for duplicate checking
	paths := map[string]string{}
	for _, d := range config.Storage.Directories {
		path, err := s.JoinPath(d.Path)
		if err != nil {
			return nil, err
		}
		if existing, ok := paths[path]; ok {
			return nil, fmt.Errorf("Directory at %s resolved to %s after symlink chasing, but another entry with path %s also resolves there",
				d.Path, path, existing)
		}
		paths[path] = d.Path
		d.Path = path
		entries = append(entries, dirEntry(d))
	}

	for _, f := range config.Storage.Files {
		path, err := s.JoinPath(f.Path)
		if err != nil {
			return nil, err
		}
		if existing, ok := paths[path]; ok {
			return nil, fmt.Errorf("File at %s resolved to %s after symlink chasing, but another entry with path %s also resolves there",
				f.Path, path, existing)
		}
		paths[path] = f.Path
		f.Path = path
		entries = append(entries, fileEntry(f))
	}

	hardlinks := []filesystemEntry{}
	for _, l := range config.Storage.Links {
		path, err := s.JoinPath(l.Path)
		if err != nil {
			return nil, err
		}
		if existing, ok := paths[path]; ok {
			return nil, fmt.Errorf("Link at %s resolved to %s after symlink chasing, but another entry with path %s also resolves there",
				l.Path, path, existing)
		}
		paths[path] = l.Path
		l.Path = path
		if l.Hard != nil && *l.Hard {
			hardlinks = append(hardlinks, linkEntry(l))
		} else {
			entries = append(entries, linkEntry(l))
		}

	}
	sort.Slice(entries, func(i, j int) bool { return util.Depth(entries[i].node().Path) < util.Depth(entries[j].node().Path) })

	// Append all the hard links to the list after sorting. This allows
	// Ignition to create hard links to files that are deeper than the hard
	// link. For reference: https://github.com/coreos/ignition/issues/800
	entries = append(entries, hardlinks...)

	return entries, nil
}

func (s *stage) removePathOnOverwrite(e filesystemEntry) error {
	if e.node().Overwrite != nil && *e.node().Overwrite {
		return os.RemoveAll(e.node().Path)
	}
	return nil
}

func (s *stage) relabelDirsForFile(path string) error {
	if !s.relabeling() {
		return nil
	}

	missing_dir, err := util.FindFirstMissingDirForFile(path)
	if err != nil {
		return err
	}

	// trim off prefix since this needs to be relative to the sysroot
	s.relabel(missing_dir[len(s.DestDir):])
	return nil
}

// createEntries creates any files or directories listed for the filesystem in Storage.{Files,Directories}.
func (s *stage) createEntries(entries []filesystemEntry) error {
	s.Logger.PushPrefix("createFiles")
	defer s.Logger.PopPrefix()

	for _, e := range entries {
		path := e.node().Path
		if !strings.HasPrefix(path, s.DestDir) {
			panic(fmt.Sprintf("Entry path %s isn't under prefix %s", path, s.DestDir))
		}

		if err := s.relabelDirsForFile(path); err != nil {
			return fmt.Errorf("error relabeling paths for %s: %v", path, err)
		}
		if err := s.removePathOnOverwrite(e); err != nil {
			return fmt.Errorf("error removing existing file %s: %v", path, err)
		}
		if err := e.create(s.Logger, s.Util); err != nil {
			return fmt.Errorf("error creating %s: %v", path, err)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package fusefrontend

// FUSE operations Truncate and Allocate on file handles
// i.e. ftruncate and fallocate

import (
	"log"
	"sync"
	"syscall"

	"github.com/hanwen/go-fuse/fuse"

	"github.com/rfjakob/gocryptfs/internal/syscallcompat"
	"github.com/rfjakob/gocryptfs/internal/tlog"
)

// FALLOC_DEFAULT is a "normal" fallocate operation
const FALLOC_DEFAULT = 0x00

// FALLOC_FL_KEEP_SIZE allocates disk space while not modifying the file size
const FALLOC_FL_KEEP_SIZE = 0x01

// Only warn once
var allocateWarnOnce sync.Once

// Allocate - FUSE call for fallocate(2)
//
// mode=FALLOC_FL_KEEP_SIZE is implemented directly.
//
// mode=FALLOC_DEFAULT is implemented as a two-step process:
//
//   (1) Allocate the space using FALLOC_FL_KEEP_SIZE
//   (2) Set the file size using ftruncate (via truncateGrowFile)
//
// This allows us to reuse the file grow mechanics from Truncate as they are
// complicated and hard to get right.
//
// Other modes (hole punching, zeroing) are not supported.
func (f *File) Allocate(off uint64, sz uint64, mode uint32) fuse.Status {
	if mode != FALLOC_DEFAULT && mode != FALLOC_FL_KEEP_SIZE {
		f := func() {
			tlog.Warn.Printf("fallocate: only mode 0 (default) and 1 (keep size) are supported")
		}
		allocateWarnOnce.Do(f)
		return fuse.Status(syscall.EOPNOTSUPP)
	}

	f.fdLock.RLock()
	defer f.fdLock.RUnlock()
	if f.released {
		return fuse.EBADF
	}
	f.fileTableEntry.ContentLock.Lock()
	defer f.fileTableEntry.ContentLock.Unlock()

	blocks := f.contentEnc.ExplodePlainRange(off, sz)
	firstBlock := blocks[0]
	lastBlock := blocks[len(blocks)-1]

	// Step (1): Allocate the space the user wants using FALLOC_FL_KEEP_SIZE.
	// This will fill file holes and/or allocate additional space past the end of
	// the file.
	cipherOff := firstBlock.BlockCipherOff()
	cipherSz := lastBlock.BlockCipherOff() - cipherOff +
		f.contentEnc.PlainSizeToCipherSize(lastBlock.Skip+lastBlock.Length)
	err := syscallcompat.Fallocate(f.intFd(), FALLOC_FL_KEEP_SIZE, int64(cipherOff), int64(cipherSz))
	tlog.Debug.Printf("Allocate off=%d sz=%d mode=%x cipherOff=%d cipherSz=%d\n",
		off, sz, mode, cipherOff, cipherSz)
	if err != nil {
		return fuse.ToStatus(err)
	}
	if mode == FALLOC_FL_KEEP_SIZE {
		// The user did not want to change the apparent size. We are done.
		return fuse.OK
	}
	// Step (2): Grow the apparent file size
	// We need the old file size to determine if we are growing the file at all.
	newPlainSz := off + sz
	oldPlainSz, err := f.statPlainSize()
	if err != nil {
		return fuse.ToStatus(err)
	}
	if newPlainSz <= oldPlainSz {
		// The new size is smaller (or equal). Fallocate with mode = 0 never
		// truncates a file, so we are done.
		return fuse.OK
	}
	// The file grows. The space has already been allocated in (1), so what is
	// left to do is to pad the first and last block and call truncate.
	// truncateGrowFile does just that.
	return f.truncateGrowFile(oldPlainSz, newPlainSz)
}

// Truncate - FUSE call
func (f *File) Truncate(newSize uint64) fuse.Status {
	f.fdLock.RLock()
	defer f.fdLock.RUnlock()
	if f.released {
		// The file descriptor has been closed concurrently.
		tlog.Warn.Printf("ino%d fh%d: Truncate on released file", f.qIno.Ino, f.intFd())
		return fuse.EBADF
	}
	f.fileTableEntry.ContentLock.Lock()
	defer f.fileTableEntry.ContentLock.Unlock()
	var err error
	// Common case first: Truncate to zero
	if newSize == 0 {
		err = syscall.Ftruncate(int(f.fd.Fd()), 0)
		if err != nil {
			tlog.Warn.Printf("ino%d fh%d: Ftruncate(fd, 0) returned error: %v", f.qIno.Ino, f.intFd(), err)
			return fuse.ToStatus(err)
		}
		// Truncate to zero kills the file header
		f.fileTableEntry.HeaderLock.Lock()
		f.fileTableEntry.ID = nil
		f.fileTableEntry.HeaderLock.Unlock()
		return fuse.OK
	}
	// We need the old file size to determine if we are growing or shrinking
	// the file
	oldSize, err := f.statPlainSize()
	if err != nil {
		return fuse.ToStatus(err)
	}

	oldB := float32(oldSize) / float32(f.contentEnc.PlainBS())
	newB := float32(newSize) / float32(f.contentEnc.PlainBS())
	tlog.Debug.Printf("ino%d: FUSE Truncate from %.2f to %.2f blocks (%d to %d bytes)", f.qIno.Ino, oldB, newB, oldSize, newSize)

	// File size stays the same - nothing to do
	if newSize == oldSize {
		return fuse.OK
	}
	// File grows
	if newSize > oldSize {
		return f.truncateGrowFile(oldSize, newSize)
	}

	// File shrinks
	blockNo := f.contentEnc.PlainOffToBlockNo(newSize)
	cipherOff := f.contentEnc.BlockNoToCipherOff(blockNo)
	plainOff := f.contentEnc.BlockNoToPlainOff(blockNo)
	lastBlockLen := newSize - plainOff
	var data []byte
	if lastBlockLen > 0 {
		var status fuse.Status
		data, status = f.doRead(nil, plainOff, lastBlockLen)
		if status != fuse.OK {
			tlog.Warn.Printf("Truncate: shrink doRead returned error: %v", err)
			return status
		}
	}
	// Truncate down to the last complete block
	err = syscall.Ftruncate(int(f.fd.Fd()), int64(cipherOff))
	if err != nil {
		tlog.Warn.Printf("Truncate: shrink Ftruncate returned error: %v", err)
		return fuse.ToStatus(err)
	}
	// Append partial block
	if lastBlockLen > 0 {
		_, status := f.doWrite(data, int64(plainOff))
		return status
	}
	return fuse.OK
}

// statPlainSize stats the file and returns the plaintext size
func (f *File) statPlainSize() (uint64, error) {
	fi, err := f.fd.Stat()
	if err != nil {
		tlog.Warn.Printf("ino%d fh%d: statPlainSize: %v", f.qIno.Ino, f.intFd(), err)
		return 0, err
	}
	cipherSz := uint64(fi.Size())
	plainSz := uint64(f.contentEnc.CipherSizeToPlainSize(cipherSz))
	return plainSz, nil
}

// truncateGrowFile extends a file using seeking or ftruncate performing RMW on
// the first and last block as necessary. New blocks in the middle become
// file holes unless they have been fallocate()'d beforehand.
func (f *File) truncateGrowFile(oldPlainSz uint64, newPlainSz uint64) fuse.Status {
	if newPlainSz <= oldPlainSz {
		log.Panicf("BUG: newSize=%d <= oldSize=%d", newPlainSz, oldPlainSz)
	}
	var n1 uint64
	if oldPlainSz > 0 {
		n1 = f.contentEnc.PlainOffToBlockNo(oldPlainSz - 1)
	}
	newEOFOffset := newPlainSz - 1
	n2 := f.contentEnc.PlainOffToBlockNo(newEOFOffset)
	// The file is grown within one block, no need to pad anything.
	// Write a single zero to the last byte and let doWrite figure out the RMW.
	if n1 == n2 {
		buf := make([]byte, 1)
		_, status := f.doWrite(buf, int64(newEOFOffset))
		return status
	}
	// The truncate creates at least one new block.
	//
	// Make sure the old last block is padded to the block boundary. This call
	// is a no-op if it is already block-aligned.
	f.zeroPad(oldPlainSz)
	// The new size is block-aligned. In this case we can do everything ourselves
	// and avoid the call to doWrite.
	if newPlainSz%f.contentEnc.PlainBS() == 0 {
		// The file was empty, so it did not have a header. Create one.
		if oldPlainSz == 0 {
			f.fileTableEntry.HeaderLock.Lock()
			defer f.fileTableEntry.HeaderLock.Unlock()
			id, err := f.createHeader()
			if err != nil {
				return fuse.ToStatus(err)
			}
			f.fileTableEntry.ID = id
		}
		cSz := int64(f.contentEnc.PlainSizeToCipherSize(newPlainSz))
		err := syscall.Ftruncate(f.intFd(), cSz)
		if err != nil {
			tlog.Warn.Printf("Truncate: grow Ftruncate returned error: %v", err)
		}
		return fuse.ToStatus(err)
	}
	// The new size is NOT aligned, so we need to write a partial block.
	// Write a single zero to the last byte and let doWrite figure it out.
	buf := make([]byte, 1)
	_, status := f.doWrite(buf, int64(newEOFOffset))
	return status
}
<commit_msg>fusefronted: downgrade fallocate message severity<commit_after>package fusefrontend

// FUSE operations Truncate and Allocate on file handles
// i.e. ftruncate and fallocate

import (
	"log"
	"sync"
	"syscall"

	"github.com/hanwen/go-fuse/fuse"

	"github.com/rfjakob/gocryptfs/internal/syscallcompat"
	"github.com/rfjakob/gocryptfs/internal/tlog"
)

// FALLOC_DEFAULT is a "normal" fallocate operation
const FALLOC_DEFAULT = 0x00

// FALLOC_FL_KEEP_SIZE allocates disk space while not modifying the file size
const FALLOC_FL_KEEP_SIZE = 0x01

// Only warn once
var allocateWarnOnce sync.Once

// Allocate - FUSE call for fallocate(2)
//
// mode=FALLOC_FL_KEEP_SIZE is implemented directly.
//
// mode=FALLOC_DEFAULT is implemented as a two-step process:
//
//   (1) Allocate the space using FALLOC_FL_KEEP_SIZE
//   (2) Set the file size using ftruncate (via truncateGrowFile)
//
// This allows us to reuse the file grow mechanics from Truncate as they are
// complicated and hard to get right.
//
// Other modes (hole punching, zeroing) are not supported.
func (f *File) Allocate(off uint64, sz uint64, mode uint32) fuse.Status {
	if mode != FALLOC_DEFAULT && mode != FALLOC_FL_KEEP_SIZE {
		f := func() {
			tlog.Info.Printf("fallocate: only mode 0 (default) and 1 (keep size) are supported")
		}
		allocateWarnOnce.Do(f)
		return fuse.Status(syscall.EOPNOTSUPP)
	}

	f.fdLock.RLock()
	defer f.fdLock.RUnlock()
	if f.released {
		return fuse.EBADF
	}
	f.fileTableEntry.ContentLock.Lock()
	defer f.fileTableEntry.ContentLock.Unlock()

	blocks := f.contentEnc.ExplodePlainRange(off, sz)
	firstBlock := blocks[0]
	lastBlock := blocks[len(blocks)-1]

	// Step (1): Allocate the space the user wants using FALLOC_FL_KEEP_SIZE.
	// This will fill file holes and/or allocate additional space past the end of
	// the file.
	cipherOff := firstBlock.BlockCipherOff()
	cipherSz := lastBlock.BlockCipherOff() - cipherOff +
		f.contentEnc.PlainSizeToCipherSize(lastBlock.Skip+lastBlock.Length)
	err := syscallcompat.Fallocate(f.intFd(), FALLOC_FL_KEEP_SIZE, int64(cipherOff), int64(cipherSz))
	tlog.Debug.Printf("Allocate off=%d sz=%d mode=%x cipherOff=%d cipherSz=%d\n",
		off, sz, mode, cipherOff, cipherSz)
	if err != nil {
		return fuse.ToStatus(err)
	}
	if mode == FALLOC_FL_KEEP_SIZE {
		// The user did not want to change the apparent size. We are done.
		return fuse.OK
	}
	// Step (2): Grow the apparent file size
	// We need the old file size to determine if we are growing the file at all.
	newPlainSz := off + sz
	oldPlainSz, err := f.statPlainSize()
	if err != nil {
		return fuse.ToStatus(err)
	}
	if newPlainSz <= oldPlainSz {
		// The new size is smaller (or equal). Fallocate with mode = 0 never
		// truncates a file, so we are done.
		return fuse.OK
	}
	// The file grows. The space has already been allocated in (1), so what is
	// left to do is to pad the first and last block and call truncate.
	// truncateGrowFile does just that.
	return f.truncateGrowFile(oldPlainSz, newPlainSz)
}

// Truncate - FUSE call
func (f *File) Truncate(newSize uint64) fuse.Status {
	f.fdLock.RLock()
	defer f.fdLock.RUnlock()
	if f.released {
		// The file descriptor has been closed concurrently.
		tlog.Warn.Printf("ino%d fh%d: Truncate on released file", f.qIno.Ino, f.intFd())
		return fuse.EBADF
	}
	f.fileTableEntry.ContentLock.Lock()
	defer f.fileTableEntry.ContentLock.Unlock()
	var err error
	// Common case first: Truncate to zero
	if newSize == 0 {
		err = syscall.Ftruncate(int(f.fd.Fd()), 0)
		if err != nil {
			tlog.Warn.Printf("ino%d fh%d: Ftruncate(fd, 0) returned error: %v", f.qIno.Ino, f.intFd(), err)
			return fuse.ToStatus(err)
		}
		// Truncate to zero kills the file header
		f.fileTableEntry.HeaderLock.Lock()
		f.fileTableEntry.ID = nil
		f.fileTableEntry.HeaderLock.Unlock()
		return fuse.OK
	}
	// We need the old file size to determine if we are growing or shrinking
	// the file
	oldSize, err := f.statPlainSize()
	if err != nil {
		return fuse.ToStatus(err)
	}

	oldB := float32(oldSize) / float32(f.contentEnc.PlainBS())
	newB := float32(newSize) / float32(f.contentEnc.PlainBS())
	tlog.Debug.Printf("ino%d: FUSE Truncate from %.2f to %.2f blocks (%d to %d bytes)", f.qIno.Ino, oldB, newB, oldSize, newSize)

	// File size stays the same - nothing to do
	if newSize == oldSize {
		return fuse.OK
	}
	// File grows
	if newSize > oldSize {
		return f.truncateGrowFile(oldSize, newSize)
	}

	// File shrinks
	blockNo := f.contentEnc.PlainOffToBlockNo(newSize)
	cipherOff := f.contentEnc.BlockNoToCipherOff(blockNo)
	plainOff := f.contentEnc.BlockNoToPlainOff(blockNo)
	lastBlockLen := newSize - plainOff
	var data []byte
	if lastBlockLen > 0 {
		var status fuse.Status
		data, status = f.doRead(nil, plainOff, lastBlockLen)
		if status != fuse.OK {
			tlog.Warn.Printf("Truncate: shrink doRead returned error: %v", err)
			return status
		}
	}
	// Truncate down to the last complete block
	err = syscall.Ftruncate(int(f.fd.Fd()), int64(cipherOff))
	if err != nil {
		tlog.Warn.Printf("Truncate: shrink Ftruncate returned error: %v", err)
		return fuse.ToStatus(err)
	}
	// Append partial block
	if lastBlockLen > 0 {
		_, status := f.doWrite(data, int64(plainOff))
		return status
	}
	return fuse.OK
}

// statPlainSize stats the file and returns the plaintext size
func (f *File) statPlainSize() (uint64, error) {
	fi, err := f.fd.Stat()
	if err != nil {
		tlog.Warn.Printf("ino%d fh%d: statPlainSize: %v", f.qIno.Ino, f.intFd(), err)
		return 0, err
	}
	cipherSz := uint64(fi.Size())
	plainSz := uint64(f.contentEnc.CipherSizeToPlainSize(cipherSz))
	return plainSz, nil
}

// truncateGrowFile extends a file using seeking or ftruncate performing RMW on
// the first and last block as necessary. New blocks in the middle become
// file holes unless they have been fallocate()'d beforehand.
func (f *File) truncateGrowFile(oldPlainSz uint64, newPlainSz uint64) fuse.Status {
	if newPlainSz <= oldPlainSz {
		log.Panicf("BUG: newSize=%d <= oldSize=%d", newPlainSz, oldPlainSz)
	}
	var n1 uint64
	if oldPlainSz > 0 {
		n1 = f.contentEnc.PlainOffToBlockNo(oldPlainSz - 1)
	}
	newEOFOffset := newPlainSz - 1
	n2 := f.contentEnc.PlainOffToBlockNo(newEOFOffset)
	// The file is grown within one block, no need to pad anything.
	// Write a single zero to the last byte and let doWrite figure out the RMW.
	if n1 == n2 {
		buf := make([]byte, 1)
		_, status := f.doWrite(buf, int64(newEOFOffset))
		return status
	}
	// The truncate creates at least one new block.
	//
	// Make sure the old last block is padded to the block boundary. This call
	// is a no-op if it is already block-aligned.
	f.zeroPad(oldPlainSz)
	// The new size is block-aligned. In this case we can do everything ourselves
	// and avoid the call to doWrite.
	if newPlainSz%f.contentEnc.PlainBS() == 0 {
		// The file was empty, so it did not have a header. Create one.
		if oldPlainSz == 0 {
			f.fileTableEntry.HeaderLock.Lock()
			defer f.fileTableEntry.HeaderLock.Unlock()
			id, err := f.createHeader()
			if err != nil {
				return fuse.ToStatus(err)
			}
			f.fileTableEntry.ID = id
		}
		cSz := int64(f.contentEnc.PlainSizeToCipherSize(newPlainSz))
		err := syscall.Ftruncate(f.intFd(), cSz)
		if err != nil {
			tlog.Warn.Printf("Truncate: grow Ftruncate returned error: %v", err)
		}
		return fuse.ToStatus(err)
	}
	// The new size is NOT aligned, so we need to write a partial block.
	// Write a single zero to the last byte and let doWrite figure it out.
	buf := make([]byte, 1)
	_, status := f.doWrite(buf, int64(newEOFOffset))
	return status
}
<|endoftext|>


<commit_before>package main

type Bytecode uint8

const (
	BC_PUTSELF Bytecode = iota
	BC_PUTNIL
	BC_PUTOBJ
	BC_PUTTRUE
	BC_PUTFALSE
	BC_SETLOCAL
	BC_GETLOCAL
	BC_SETGLOBAL
	BC_GETGLOBAL
	BC_SETSYMBOL
	BC_GETSYMBOL
	BC_SETCONST
	BC_GETCONST
	BC_SETIVAR
	BC_GETIVAR
	BC_SETCVAR
	BC_GETCVAR
	BC_SEND
	BC_JUMP
)

type Instruction struct {
	inst_seq  int
	inst_type Bytecode
	obj       Object
	// block     Instruction[]
	argc int
}

func (vm *GobiesVM) AddInstruction(inst Bytecode, obj Object) {
	new_inst := &Instruction{inst_type: inst, inst_seq: len(vm.instList)}
	if obj != nil {
		new_inst.obj = obj
	}
	vm.instList = append(vm.instList, new_inst)
}

func (VM *GobiesVM) compile(node *AST) {

	switch node.Type {
	case NODE_ROOT:
		VM.compile(node.args[0])
	case NODE_LIST:
	case NODE_BLOCK:
		body := node.args[0]
		params := node.args[1]
		inst_last_idx := len(VM.instList)
		if params != nil { // Block has params
			// TODO:
		} else {

		}
		VM.compile(body)

		// block_inst_len := len(VM.instList) - inst_last_idx
		// block := VM.instList[inst_last_idx:-1]
		VM.instList = VM.instList[0:inst_last_idx]
		// VM.instList[inst_last_idx].block = block
	case NODE_VALUE:
		// NODE_VALUE can only be either NUMBER or SYMBOL
		astval := node.args[0]
		if len(astval.value.str) != 0 { // SYMBOL
			VM.AddInstruction(BC_SETSYMBOL, &RString{val: RValue{str: astval.value.str}})
		} else { // NUMBER
			VM.AddInstruction(BC_PUTOBJ, &RFixnum{val: RValue{fixnum: node.value.numeric}})
		}
	case NODE_ASTVAL:
		if len(node.value.str) != 0 {
			VM.AddInstruction(BC_PUTOBJ, &RString{val: RValue{str: node.value.str}})
		} else {
			VM.AddInstruction(BC_PUTOBJ, &RFixnum{val: RValue{fixnum: node.value.numeric}})
		}
	case NODE_STRING:
		astval := node.args[0]
		VM.AddInstruction(BC_PUTOBJ, &RString{val: RValue{str: astval.value.str}})
	case NODE_ASSIGN:
		// Set local variable
		astval := node.args[0]
		VM.compile(node.args[1])
		VM.AddInstruction(BC_SETLOCAL, &RString{val: RValue{str: astval.value.str}})
	case NODE_ARG:
		head := node.args[0]
		for {
			VM.compile(head)
			if head.next == nil {
				break
			}
			head = head.next
		}
	case NODE_SEND: // Appears in Call, AsgnCall, SpecCall and Method.
		rcv := node.args[0]
		msg := node.args[1]
		argc := 0
		block := node.args[2]

		if rcv == nil && block == nil {
			if msg.args[1] == nil { // Local variable access
				// TODO: Local method call without any argument
				VM.AddInstruction(BC_GETLOCAL, &RString{val: RValue{str: msg.args[0].value.str}})
			} else { // Calling local methods with one or more arguments
				if msg.args[1] != nil {
					VM.AddInstruction(BC_PUTSELF, nil)
					VM.compile(msg.args[1])
					argc = msg.args[1].args[0].length
				}
				VM.AddInstruction(BC_SEND, &RString{val: RValue{str: msg.args[0].value.str}})
				VM.instList[len(VM.instList)-1].argc = argc
			}
		} else {
			if rcv == nil { // SELF
				VM.AddInstruction(BC_PUTSELF, nil)
			} else {
				VM.compile(rcv)
			}

			if msg.args[1] != nil {
				VM.compile(msg.args[1])
			}

			if block != nil {
				VM.compile(block)
			}

			VM.AddInstruction(BC_SEND, &RString{val: RValue{str: msg.args[0].value.str}})
			VM.instList[len(VM.instList)-1].argc = argc
		}
	case NODE_MSG:
		VM.compile(node.args[0])
		if node.args[1] != nil { // Arguments
			VM.compile(node.args[1])
		}
	case NODE_IF:
	case NODE_UNLESS:
	case NODE_AND:
	case NODE_OR:
	case NODE_WHILE:
	case NODE_UNTIL:
	case NODE_TRUE:
		VM.AddInstruction(BC_PUTTRUE, nil)
	case NODE_FALSE:
		VM.AddInstruction(BC_PUTFALSE, nil)
	case NODE_NIL:
		VM.AddInstruction(BC_PUTNIL, nil)
	case NODE_SELF:
		VM.AddInstruction(BC_PUTSELF, nil)
	case NODE_LEAVE:
	case NODE_RETURN:
	case NODE_BREAK:
	case NODE_YIELD:
	case NODE_DEF:
	case NODE_METHOD: // Appears in method definition
	case NODE_PARAM: // Appears in method parameters and blocks
	case NODE_CLASS:
	case NODE_MODULE:
	case NODE_CONST:
		astval := node.args[0]
		VM.AddInstruction(BC_GETCONST, &RString{val: RValue{str: astval.value.str}})
	case NODE_SETCONST:
		name := node.args[0]
		val := node.args[1]
		VM.compile(val)
		VM.AddInstruction(BC_SETCONST, &RString{val: RValue{str: name.value.str}})
	case NODE_ARRAY:
	case NODE_HASH:
	case NODE_RANGE:
	case NODE_GETIVAR:
		name := node.args[0]
		VM.AddInstruction(BC_GETIVAR, &RString{val: RValue{str: name.value.str}})
	case NODE_SETIVAR:
	case NODE_GETCVAR:
		name := node.args[0]
		VM.AddInstruction(BC_GETCVAR, &RString{val: RValue{str: name.value.str}})
	case NODE_SETCVAR:
	case NODE_GETGLOBAL:
	case NODE_SETGLOBAL:
	case NODE_ADD:
	case NODE_SUB:
	case NODE_LT:
	case NODE_NEG:
	case NODE_NOT:
	}
}
<commit_msg>Fixed the problem that only one statement would be compiled<commit_after>package main

type Bytecode uint8

const (
	BC_PUTSELF Bytecode = iota
	BC_PUTNIL
	BC_PUTOBJ
	BC_PUTTRUE
	BC_PUTFALSE
	BC_SETLOCAL
	BC_GETLOCAL
	BC_SETGLOBAL
	BC_GETGLOBAL
	BC_SETSYMBOL
	BC_GETSYMBOL
	BC_SETCONST
	BC_GETCONST
	BC_SETIVAR
	BC_GETIVAR
	BC_SETCVAR
	BC_GETCVAR
	BC_SEND
	BC_JUMP
)

type Instruction struct {
	inst_seq  int
	inst_type Bytecode
	obj       Object
	// block     Instruction[]
	argc int
}

func (vm *GobiesVM) AddInstruction(inst Bytecode, obj Object) {
	new_inst := &Instruction{inst_type: inst, inst_seq: len(vm.instList)}
	if obj != nil {
		new_inst.obj = obj
	}
	vm.instList = append(vm.instList, new_inst)
}

func (VM *GobiesVM) compile(node *AST) {

	switch node.Type {
	case NODE_ROOT:
		head := node.args[0]
		for head != nil {
			VM.compile(head)
			head = head.next
		}
	case NODE_LIST:
	case NODE_BLOCK:
		body := node.args[0]
		params := node.args[1]
		inst_last_idx := len(VM.instList)
		if params != nil { // Block has params
			// TODO:
		} else {

		}
		VM.compile(body)

		// block_inst_len := len(VM.instList) - inst_last_idx
		// block := VM.instList[inst_last_idx:-1]
		VM.instList = VM.instList[0:inst_last_idx]
		// VM.instList[inst_last_idx].block = block
	case NODE_VALUE:
		// NODE_VALUE can only be either NUMBER or SYMBOL
		astval := node.args[0]
		if len(astval.value.str) != 0 { // SYMBOL
			VM.AddInstruction(BC_SETSYMBOL, &RString{val: RValue{str: astval.value.str}})
		} else { // NUMBER
			VM.AddInstruction(BC_PUTOBJ, &RFixnum{val: RValue{fixnum: node.value.numeric}})
		}
	case NODE_ASTVAL:
		if len(node.value.str) != 0 {
			VM.AddInstruction(BC_PUTOBJ, &RString{val: RValue{str: node.value.str}})
		} else {
			VM.AddInstruction(BC_PUTOBJ, &RFixnum{val: RValue{fixnum: node.value.numeric}})
		}
	case NODE_STRING:
		astval := node.args[0]
		VM.AddInstruction(BC_PUTOBJ, &RString{val: RValue{str: astval.value.str}})
	case NODE_ASSIGN:
		// Set local variable
		astval := node.args[0]
		VM.compile(node.args[1])
		VM.AddInstruction(BC_SETLOCAL, &RString{val: RValue{str: astval.value.str}})
	case NODE_ARG:
		head := node.args[0]
		for {
			VM.compile(head)
			if head.next == nil {
				break
			}
			head = head.next
		}
	case NODE_SEND: // Appears in Call, AsgnCall, SpecCall and Method.
		rcv := node.args[0]
		msg := node.args[1]
		argc := 0
		block := node.args[2]

		if rcv == nil && block == nil {
			if msg.args[1] == nil { // Local variable access
				// TODO: Local method call without any argument
				VM.AddInstruction(BC_GETLOCAL, &RString{val: RValue{str: msg.args[0].value.str}})
			} else { // Calling local methods with one or more arguments
				if msg.args[1] != nil {
					VM.AddInstruction(BC_PUTSELF, nil)
					VM.compile(msg.args[1])
					argc = msg.args[1].args[0].length
				}
				VM.AddInstruction(BC_SEND, &RString{val: RValue{str: msg.args[0].value.str}})
				VM.instList[len(VM.instList)-1].argc = argc
			}
		} else {
			if rcv == nil { // SELF
				VM.AddInstruction(BC_PUTSELF, nil)
			} else {
				VM.compile(rcv)
			}

			if msg.args[1] != nil {
				VM.compile(msg.args[1])
			}

			if block != nil {
				VM.compile(block)
			}

			VM.AddInstruction(BC_SEND, &RString{val: RValue{str: msg.args[0].value.str}})
			VM.instList[len(VM.instList)-1].argc = argc
		}
	case NODE_MSG:
		VM.compile(node.args[0])
		if node.args[1] != nil { // Arguments
			VM.compile(node.args[1])
		}
	case NODE_IF:
	case NODE_UNLESS:
	case NODE_AND:
	case NODE_OR:
	case NODE_WHILE:
	case NODE_UNTIL:
	case NODE_TRUE:
		VM.AddInstruction(BC_PUTTRUE, nil)
	case NODE_FALSE:
		VM.AddInstruction(BC_PUTFALSE, nil)
	case NODE_NIL:
		VM.AddInstruction(BC_PUTNIL, nil)
	case NODE_SELF:
		VM.AddInstruction(BC_PUTSELF, nil)
	case NODE_LEAVE:
	case NODE_RETURN:
	case NODE_BREAK:
	case NODE_YIELD:
	case NODE_DEF:
	case NODE_METHOD: // Appears in method definition
	case NODE_PARAM: // Appears in method parameters and blocks
	case NODE_CLASS:
	case NODE_MODULE:
	case NODE_CONST:
		astval := node.args[0]
		VM.AddInstruction(BC_GETCONST, &RString{val: RValue{str: astval.value.str}})
	case NODE_SETCONST:
		name := node.args[0]
		val := node.args[1]
		VM.compile(val)
		VM.AddInstruction(BC_SETCONST, &RString{val: RValue{str: name.value.str}})
	case NODE_ARRAY:
	case NODE_HASH:
	case NODE_RANGE:
	case NODE_GETIVAR:
		name := node.args[0]
		VM.AddInstruction(BC_GETIVAR, &RString{val: RValue{str: name.value.str}})
	case NODE_SETIVAR:
	case NODE_GETCVAR:
		name := node.args[0]
		VM.AddInstruction(BC_GETCVAR, &RString{val: RValue{str: name.value.str}})
	case NODE_SETCVAR:
	case NODE_GETGLOBAL:
	case NODE_SETGLOBAL:
	case NODE_ADD:
	case NODE_SUB:
	case NODE_LT:
	case NODE_NEG:
	case NODE_NOT:
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Mender Software AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
package main

import (
	"bytes"
	"encoding/json"
	"net/http"

	"github.com/mendersoftware/log"
	"github.com/pkg/errors"
)

type InventoryReporter interface {
	Submit(api ApiRequester, server string, data interface{}) error
}

type InventoryClient struct {
}

func NewInventoryClient() InventoryReporter {
	return &InventoryClient{}
}

// Report status information to the backend
func (i *InventoryClient) Submit(api ApiRequester, url string, data interface{}) error {
	req, err := makeInventorySubmitRequest(url, data)
	if err != nil {
		return errors.Wrapf(err, "failed to prepare inventory submit request")
	}

	r, err := api.Do(req)
	if err != nil {
		log.Error("failed to submit inventory data: ", err)
		return errors.Wrapf(err, "inventory submit failed")
	}

	defer r.Body.Close()

	if r.StatusCode != http.StatusOK {
		log.Errorf("got unexpected HTTP status when submitting to inventory: %v", r.StatusCode)
		return errors.Errorf("inventory submit failed, bad status %v", r.StatusCode)
	}
	log.Debugf("inventory update sent, response %v", r)

	return nil
}

func makeInventorySubmitRequest(server string, data interface{}) (*http.Request, error) {
	url := buildApiURL(server, "/inventory/device/attributes")

	out := &bytes.Buffer{}
	enc := json.NewEncoder(out)
	enc.Encode(&data)

	hreq, err := http.NewRequest(http.MethodPatch, url, out)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to create inventory HTTP request")
	}

	hreq.Header.Add("Content-Type", "application/json")
	return hreq, nil
}
<commit_msg>client_inventory: rename interface to InventorySubmitter<commit_after>// Copyright 2016 Mender Software AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
package main

import (
	"bytes"
	"encoding/json"
	"net/http"

	"github.com/mendersoftware/log"
	"github.com/pkg/errors"
)

type InventorySubmitter interface {
	Submit(api ApiRequester, server string, data interface{}) error
}

type InventoryClient struct {
}

func NewInventoryClient() InventorySubmitter {
	return &InventoryClient{}
}

// Report status information to the backend
func (i *InventoryClient) Submit(api ApiRequester, url string, data interface{}) error {
	req, err := makeInventorySubmitRequest(url, data)
	if err != nil {
		return errors.Wrapf(err, "failed to prepare inventory submit request")
	}

	r, err := api.Do(req)
	if err != nil {
		log.Error("failed to submit inventory data: ", err)
		return errors.Wrapf(err, "inventory submit failed")
	}

	defer r.Body.Close()

	if r.StatusCode != http.StatusOK {
		log.Errorf("got unexpected HTTP status when submitting to inventory: %v", r.StatusCode)
		return errors.Errorf("inventory submit failed, bad status %v", r.StatusCode)
	}
	log.Debugf("inventory update sent, response %v", r)

	return nil
}

func makeInventorySubmitRequest(server string, data interface{}) (*http.Request, error) {
	url := buildApiURL(server, "/inventory/device/attributes")

	out := &bytes.Buffer{}
	enc := json.NewEncoder(out)
	enc.Encode(&data)

	hreq, err := http.NewRequest(http.MethodPatch, url, out)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to create inventory HTTP request")
	}

	hreq.Header.Add("Content-Type", "application/json")
	return hreq, nil
}
<|endoftext|>


<commit_before>package objectserver

import (
	"io"
	"time"

	"github.com/Symantec/Dominator/lib/hash"
)

type FullObjectServer interface {
	DeleteObject(hashVal hash.Hash) error
	ObjectServer
	LastMutationTime() time.Time
	ListObjectSizes() map[hash.Hash]uint64
	ListObjects() []hash.Hash
	NumObjects() uint64
}

type AddCallback func(hashVal hash.Hash, length uint64, isNew bool)

type AddCallbackSetter interface {
	SetAddCallback(callback AddCallback)
}

type GarbageCollector func(bytesToDelete uint64) (
	bytesDeleted uint64, err error)

type GarbageCollectorSetter interface {
	SetGarbageCollector(gc GarbageCollector)
}

type ObjectGetter interface {
	GetObject(hashVal hash.Hash) (uint64, io.ReadCloser, error)
}

type ObjectsGetter interface {
	GetObjects(hashes []hash.Hash) (ObjectsReader, error)
}

type ObjectsReader interface {
	Close() error
	NextObject() (uint64, io.ReadCloser, error)
}

type ObjectServer interface {
	AddObject(reader io.Reader, length uint64, expectedHash *hash.Hash) (
		hash.Hash, bool, error)
	CheckObjects(hashes []hash.Hash) ([]uint64, error)
	ObjectGetter
	ObjectsGetter
}

type StashingObjectServer interface {
	CommitObject(hash.Hash) error
	DeleteStashedObject(hashVal hash.Hash) error
	ObjectServer
	StashOrVerifyObject(io.Reader, uint64, *hash.Hash) (
		hash.Hash, []byte, error)
}

func GetObject(objSrv ObjectServer, hashVal hash.Hash) (
	uint64, io.ReadCloser, error) {
	return getObject(objSrv, hashVal)
}
<commit_msg>Add lib/objectserver.ObjectsChecker interface type.<commit_after>package objectserver

import (
	"io"
	"time"

	"github.com/Symantec/Dominator/lib/hash"
)

type FullObjectServer interface {
	DeleteObject(hashVal hash.Hash) error
	ObjectServer
	LastMutationTime() time.Time
	ListObjectSizes() map[hash.Hash]uint64
	ListObjects() []hash.Hash
	NumObjects() uint64
}

type AddCallback func(hashVal hash.Hash, length uint64, isNew bool)

type AddCallbackSetter interface {
	SetAddCallback(callback AddCallback)
}

type GarbageCollector func(bytesToDelete uint64) (
	bytesDeleted uint64, err error)

type GarbageCollectorSetter interface {
	SetGarbageCollector(gc GarbageCollector)
}

type ObjectGetter interface {
	GetObject(hashVal hash.Hash) (uint64, io.ReadCloser, error)
}

type ObjectsChecker interface {
	CheckObjects(hashes []hash.Hash) ([]uint64, error)
}

type ObjectsGetter interface {
	GetObjects(hashes []hash.Hash) (ObjectsReader, error)
}

type ObjectsReader interface {
	Close() error
	NextObject() (uint64, io.ReadCloser, error)
}

type ObjectServer interface {
	AddObject(reader io.Reader, length uint64, expectedHash *hash.Hash) (
		hash.Hash, bool, error)
	ObjectGetter
	ObjectsChecker
	ObjectsGetter
}

type StashingObjectServer interface {
	CommitObject(hash.Hash) error
	DeleteStashedObject(hashVal hash.Hash) error
	ObjectServer
	StashOrVerifyObject(io.Reader, uint64, *hash.Hash) (
		hash.Hash, []byte, error)
}

func GetObject(objSrv ObjectServer, hashVal hash.Hash) (
	uint64, io.ReadCloser, error) {
	return getObject(objSrv, hashVal)
}
<|endoftext|>