<commit_before>package main

import (
	"database/sql"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"log"
	"time"

	"github.com/hashicorp/raft"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/robustirc/robustirc/ircserver"
	"github.com/robustirc/robustirc/raft_store"
	"github.com/robustirc/robustirc/types"
	"github.com/sorcix/irc"
	"github.com/stapelberg/glog"
)

var (
	compactionDurations = prometheus.NewSummary(prometheus.SummaryOpts{
		Name: "compaction_duration_milliseconds",
		Help: "A summary of compaction wall-clock time, in milliseconds.",
	})
)

func init() {
	prometheus.MustRegister(compactionDurations)
}

// returnedOnlyErrors returns true if and only if |msgid| resulted in at least
// one error and not a single non-error IRC reply.
func returnedOnlyErrors(msgid types.RobustId) bool {
	vmsgs, _ := ircServer.Get(msgid)
	for _, msg := range vmsgs {
		ircmsg := irc.ParseMessage(msg.Data)
		if ircmsg == nil {
			glog.Errorf("Output message not parsable\n")
			continue
		}
		if !ircserver.ErrorCodes[ircmsg.Command] {
			return false
		}
	}
	return len(vmsgs) > 0
}

type robustSnapshot struct {
	firstIndex            uint64
	lastIndex             uint64
	store                 *raft_store.LevelDBStore
	del                   map[uint64]types.RobustId
	servers               map[int64]bool
	idToIdx               map[int64]uint64
	compactionEnd         time.Time
	skipDeletionForCanary bool
}

func (s *robustSnapshot) Persist(sink raft.SnapshotSink) error {
	log.Printf("Filtering and writing up to %d indexes\n", s.lastIndex-s.firstIndex)

	// Get a timestamp and keep it constant, so that we only compact messages
	// older than n days from compactionStart. If we used time.Since, new
	// messages would pour into the window on every compaction round, possibly
	// making the compaction never converge.
	compactionStart := time.Now()
	log.Printf("compactionStart %s\n", compactionStart.String())
	if *canaryCompactionStart > 0 {
		compactionStart = time.Unix(0, *canaryCompactionStart)
		log.Printf("compactionStart %s (overridden with -canary_compaction_start)\n", compactionStart.String())
	}

	s.compactionEnd = compactionStart.Add(-7 * 24 * time.Hour)

	db := ircServer.CompactionDatabase.DB

	const windows = `
CREATE VIEW allMessagesWin AS SELECT * FROM allMessages WHERE msgid < %d;
CREATE VIEW createSessionWin AS SELECT * FROM createSession WHERE msgid < %d;
CREATE VIEW deleteSessionWin AS SELECT * FROM deleteSession WHERE msgid < %d;
`

	if _, err := db.Exec(
		fmt.Sprintf(windows,
			s.compactionEnd.UnixNano(),
			s.compactionEnd.UnixNano(),
			s.compactionEnd.UnixNano())); err != nil {
		return err
	}

	for _, cmd := range ircserver.Commands {
		if cmd.CompactionPrepareViews == nil {
			continue
		}
		if err := cmd.CompactionPrepareViews(db, s.compactionEnd); err != nil {
			return err
		}
	}

	// First pass: build up s.idToIdx
	iterator := s.store.GetBulkIterator(s.firstIndex, s.lastIndex+1)
	defer iterator.Release()
	available := iterator.First()
	for available {
		var nlog raft.Log
		if err := iterator.Error(); err != nil {
			return err
		}
		i := binary.BigEndian.Uint64(iterator.Key())
		value := iterator.Value()
		if err := json.Unmarshal(value, &nlog); err != nil {
			glog.Errorf("Skipping log entry %d because of a JSON unmarshaling error: %v", i, err)
			available = iterator.Next()
			continue
		}
		available = iterator.Next()

		if nlog.Type != raft.LogCommand {
			return fmt.Errorf("nlog.Type = %d instead of LogCommand", nlog.Type)
		}

		parsed := types.NewRobustMessageFromBytes(nlog.Data)
		if time.Unix(0, parsed.Id.Id).After(s.compactionEnd) {
			break
		}
		s.idToIdx[parsed.Id.Id] = i
		switch parsed.Type {
		case types.RobustMessageOfDeath:
			s.del[i] = parsed.Id
			continue

		case types.RobustConfig:
			// TODO: implement compaction for RobustConfig
			continue

		case types.RobustIRCToClient:
			fallthrough
		case types.RobustPing:
			fallthrough
		case types.RobustAny:
			glog.Errorf("Compaction: saw unexpected message type %v (log id %d, message id %v)\n", parsed.Type, i, parsed.Id)
			continue

		case types.RobustIRCFromClient:
			fallthrough
		case types.RobustCreateSession:
			fallthrough
		case types.RobustDeleteSession:
		// Handled in FSM.Apply()

		default:
			glog.Errorf("Compaction: saw message of unknown type %v (log id %d, message id %v)\n", parsed.Type, i, parsed.Id)
			continue
		}
	}

	// We repeatedly compact, since the result of one compaction can affect the
	// result of other compactions (see compaction_test.go for examples).
	changed := true
	pass := 0
	for changed {
		log.Printf("Compaction pass %d\n", pass)
		pass++
		var err error
		changed, err = s.compact(db)
		if err != nil {
			return err
		}
	}

	if _, err := db.Exec(`
		DROP VIEW allMessagesWin;
		DROP VIEW createSessionWin;
		DROP VIEW deleteSessionWin;
		`); err != nil {
		return err
	}

	for _, cmd := range ircserver.Commands {
		if cmd.CompactionDropViews == nil {
			continue
		}
		if err := cmd.CompactionDropViews(db); err != nil {
			return err
		}
	}

	if _, err := db.Exec("VACUUM"); err != nil {
		return err
	}

	log.Printf("Copying non-deleted messages into snapshot\n")

	snapshotBytes := 0
	available = iterator.First()
	for available {
		if err := iterator.Error(); err != nil {
			return err
		}
		i := binary.BigEndian.Uint64(iterator.Key())
		value := iterator.Value()
		if _, ok := s.del[i]; !ok {
			n, err := sink.Write(value)
			if err != nil {
				return err
			}
			snapshotBytes += n
		}
		available = iterator.Next()
	}
	log.Printf("snapshot: wrote %d bytes", snapshotBytes)

	if s.skipDeletionForCanary {
		log.Printf("Skipping deletion (canary mode)\n")
		return nil
	}

	log.Printf("Deleting compacted messages from disk\n")
	for idx, id := range s.del {
		if err := ircServer.Delete(id); err != nil {
			log.Panicf("Could not delete outputstream message: %v\n", err)
		}
		s.store.DeleteRange(idx, idx)
	}

	log.Printf("Snapshot done\n")
	compactionDurations.Observe(float64((time.Now().Sub(compactionStart)).Nanoseconds() / int64(time.Millisecond)))

	return nil
}

func (s *robustSnapshot) Release() {
}

// compact invokes all irc command’s |Compact| callbacks and runs the
// compaction for createSession and deleteSession messages.
func (s *robustSnapshot) compact(db *sql.DB) (bool, error) {
	const nonModifying = `
CREATE TABLE deleteIds AS
SELECT msgid FROM allMessagesWin WHERE irccommand = 'TOPIC' AND msgid NOT IN (SELECT msgid FROM paramsTopicWin)
UNION SELECT msgid FROM allMessagesWin WHERE irccommand = 'NICK' AND msgid NOT IN (SELECT msgid FROM paramsNickWin)
UNION SELECT msgid FROM allMessagesWin WHERE irccommand = 'USER' AND msgid NOT IN (SELECT msgid FROM paramsUserWin)
UNION SELECT msgid FROM allMessagesWin WHERE irccommand = 'JOIN' AND msgid NOT IN (SELECT msgid FROM paramsJoinWin)
UNION SELECT msgid FROM allMessagesWin WHERE irccommand = 'PART' AND msgid NOT IN (SELECT msgid FROM paramsPartWin)
UNION SELECT msgid FROM allMessagesWin WHERE irccommand = 'QUIT' AND msgid NOT IN (SELECT msgid FROM paramsQuitWin)
UNION SELECT msgid FROM allMessagesWin WHERE irccommand = 'MODE' AND msgid NOT IN (SELECT msgid FROM paramsModeWin)
UNION SELECT msgid FROM allMessagesWin WHERE irccommand = 'AWAY' AND msgid NOT IN (SELECT msgid FROM paramsAwayWin);
	`
	started := time.Now()
	if _, err := db.Exec(nonModifying); err != nil {
		return false, err
	}

	changed, err := s.markResultForCompaction(db, "non-modifying", started)
	if err != nil {
		return changed, err
	}

	if _, err := db.Exec("CREATE TABLE validCommands (command text not null)"); err != nil {
		return changed, err
	}
	validCommandInsert, err := db.Prepare("INSERT INTO validCommands (command) VALUES (?)")
	if err != nil {
		return changed, err
	}
	defer validCommandInsert.Close()

	immediatelyCompactable, err := db.Prepare("CREATE TABLE deleteIds AS SELECT msgid FROM allMessagesWin WHERE irccommand = ?")
	if err != nil {
		return changed, err
	}
	defer immediatelyCompactable.Close()

	for n, cmd := range ircserver.Commands {
		if _, err := validCommandInsert.Exec(n); err != nil {
			return changed, err
		}
		started := time.Now()
		if cmd.Compact != nil {
			if err := cmd.Compact(db); err != nil {
				return changed, err
			}
		} else if cmd.ImmediatelyCompactable {
			if _, err := immediatelyCompactable.Exec(n); err != nil {
				return changed, err
			}
		} else {
			continue
		}
		c, err := s.markResultForCompaction(db, "command "+n, started)
		if err != nil {
			return changed, err
		}
		changed = changed || c
	}

	const invalidCommands = `
	CREATE TABLE deleteIds AS SELECT msgid FROM allMessagesWin WHERE irccommand NOT IN (SELECT command FROM validCommands);
	DROP TABLE validCommands;
	`
	started = time.Now()
	_, err = db.Exec(invalidCommands)
	if err != nil {
		return changed, err
	}

	c, err := s.markResultForCompaction(db, "invalid commands", started)
	if err != nil {
		return changed, err
	}

	const stmt = `
-- Delete all deleteSession messages immediately preceded by a createSession
-- message (or commands in between which do not modify state of anything but
-- the session in question).
CREATE TABLE candidates AS
SELECT
    a.msgid AS msgid,
    a.session AS session
FROM
    (
        SELECT
            d.msgid AS msgid,
            d.session AS session,
            MAX(a.msgid) AS next_msgid
        FROM
            (SELECT msgid, session FROM deleteSessionWin
             UNION SELECT msgid, session FROM paramsQuitWin) AS d
            INNER JOIN allMessagesWin AS a
            ON (
                d.session = a.session AND
                (a.irccommand IS NULL OR
                 (a.irccommand != 'NICK' AND
                  a.irccommand != 'USER' AND
                  a.irccommand != 'PASS' AND
                  a.irccommand != 'QUIT')) AND
                a.msgid < d.msgid
            )
        GROUP BY d.msgid
    ) AS a
    INNER JOIN createSessionWin AS c
    ON (
        a.session = c.session AND
        a.next_msgid = c.msgid
    );
DELETE FROM deleteSession WHERE msgid IN (SELECT msgid FROM candidates);

INSERT INTO candidates
SELECT
    a.msgid,
    a.session
FROM
    allMessagesWin AS a
    LEFT JOIN candidates AS d
    ON (
        a.session = d.session
    )
    WHERE d.session IS NOT NULL;

CREATE TABLE deleteIds AS SELECT msgid FROM candidates;
DROP TABLE candidates;
DELETE FROM createSession WHERE msgid IN (SELECT msgid FROM deleteIds);
`

	started = time.Now()
	_, err = db.Exec(stmt)
	if err != nil {
		return changed, err
	}

	c, err = s.markResultForCompaction(db, "deleteSession, createSession", started)
	if err != nil {
		return changed, err
	}
	return changed || c, nil
}

// markResultForCompaction updates |s.del| with all messages whose ids are in
// the deleteIds table. It then removes these messages from the allMessages
// table and drops the deleteIds table.
func (s *robustSnapshot) markResultForCompaction(db *sql.DB, label string, started time.Time) (bool, error) {
	changed := false
	rows, err := db.Query("SELECT msgid FROM deleteIds")
	if err != nil {
		return changed, err
	}
	defer rows.Close()
	deleted := 0
	for rows.Next() {
		var msgid int64
		if err := rows.Scan(&msgid); err != nil {
			return changed, err
		}
		idx, ok := s.idToIdx[msgid]
		if !ok {
			return changed, fmt.Errorf("msgid=%v not in idToIdx", msgid)
		}
		s.del[idx] = types.RobustId{Id: msgid}
		changed = true
		deleted++
	}
	if err := rows.Err(); err != nil {
		return changed, err
	}
	log.Printf("%d rows deleted in %v (step %q)\n", deleted, time.Since(started), label)

	if _, err := db.Exec("DELETE FROM allMessages WHERE msgid IN (SELECT msgid FROM deleteIds)"); err != nil {
		return changed, err
	}

	_, err = db.Exec("DROP TABLE deleteIds")
	return changed, nil
}
<commit_msg>compaction: export the number of deleteSession messages<commit_after>package main

import (
	"database/sql"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"log"
	"time"

	"github.com/hashicorp/raft"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/robustirc/robustirc/ircserver"
	"github.com/robustirc/robustirc/raft_store"
	"github.com/robustirc/robustirc/types"
	"github.com/sorcix/irc"
	"github.com/stapelberg/glog"
)

var (
	compactionDurations = prometheus.NewSummary(prometheus.SummaryOpts{
		Name: "compaction_duration_milliseconds",
		Help: "A summary of compaction wall-clock time, in milliseconds.",
	})
	numDeleteSessionsWin = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "num_delete_sessions_win",
		Help: "Number of deleteSession messages in the compaction window.",
	})
)

func init() {
	prometheus.MustRegister(compactionDurations)
	prometheus.MustRegister(numDeleteSessionsWin)
}

// returnedOnlyErrors returns true if and only if |msgid| resulted in at least
// one error and not a single non-error IRC reply.
func returnedOnlyErrors(msgid types.RobustId) bool {
	vmsgs, _ := ircServer.Get(msgid)
	for _, msg := range vmsgs {
		ircmsg := irc.ParseMessage(msg.Data)
		if ircmsg == nil {
			glog.Errorf("Output message not parsable\n")
			continue
		}
		if !ircserver.ErrorCodes[ircmsg.Command] {
			return false
		}
	}
	return len(vmsgs) > 0
}

type robustSnapshot struct {
	firstIndex            uint64
	lastIndex             uint64
	store                 *raft_store.LevelDBStore
	del                   map[uint64]types.RobustId
	servers               map[int64]bool
	idToIdx               map[int64]uint64
	compactionEnd         time.Time
	skipDeletionForCanary bool
}

func (s *robustSnapshot) Persist(sink raft.SnapshotSink) error {
	log.Printf("Filtering and writing up to %d indexes\n", s.lastIndex-s.firstIndex)

	// Get a timestamp and keep it constant, so that we only compact messages
	// older than n days from compactionStart. If we used time.Since, new
	// messages would pour into the window on every compaction round, possibly
	// making the compaction never converge.
	compactionStart := time.Now()
	log.Printf("compactionStart %s\n", compactionStart.String())
	if *canaryCompactionStart > 0 {
		compactionStart = time.Unix(0, *canaryCompactionStart)
		log.Printf("compactionStart %s (overridden with -canary_compaction_start)\n", compactionStart.String())
	}

	s.compactionEnd = compactionStart.Add(-7 * 24 * time.Hour)

	db := ircServer.CompactionDatabase.DB

	const windows = `
CREATE VIEW allMessagesWin AS SELECT * FROM allMessages WHERE msgid < %d;
CREATE VIEW createSessionWin AS SELECT * FROM createSession WHERE msgid < %d;
CREATE VIEW deleteSessionWin AS SELECT * FROM deleteSession WHERE msgid < %d;
`

	if _, err := db.Exec(
		fmt.Sprintf(windows,
			s.compactionEnd.UnixNano(),
			s.compactionEnd.UnixNano(),
			s.compactionEnd.UnixNano())); err != nil {
		return err
	}

	for _, cmd := range ircserver.Commands {
		if cmd.CompactionPrepareViews == nil {
			continue
		}
		if err := cmd.CompactionPrepareViews(db, s.compactionEnd); err != nil {
			return err
		}
	}

	// First pass: build up s.idToIdx
	iterator := s.store.GetBulkIterator(s.firstIndex, s.lastIndex+1)
	defer iterator.Release()
	available := iterator.First()
	for available {
		var nlog raft.Log
		if err := iterator.Error(); err != nil {
			return err
		}
		i := binary.BigEndian.Uint64(iterator.Key())
		value := iterator.Value()
		if err := json.Unmarshal(value, &nlog); err != nil {
			glog.Errorf("Skipping log entry %d because of a JSON unmarshaling error: %v", i, err)
			available = iterator.Next()
			continue
		}
		available = iterator.Next()

		if nlog.Type != raft.LogCommand {
			return fmt.Errorf("nlog.Type = %d instead of LogCommand", nlog.Type)
		}

		parsed := types.NewRobustMessageFromBytes(nlog.Data)
		if time.Unix(0, parsed.Id.Id).After(s.compactionEnd) {
			break
		}
		s.idToIdx[parsed.Id.Id] = i
		switch parsed.Type {
		case types.RobustMessageOfDeath:
			s.del[i] = parsed.Id
			continue

		case types.RobustConfig:
			// TODO: implement compaction for RobustConfig
			continue

		case types.RobustIRCToClient:
			fallthrough
		case types.RobustPing:
			fallthrough
		case types.RobustAny:
			glog.Errorf("Compaction: saw unexpected message type %v (log id %d, message id %v)\n", parsed.Type, i, parsed.Id)
			continue

		case types.RobustIRCFromClient:
			fallthrough
		case types.RobustCreateSession:
			fallthrough
		case types.RobustDeleteSession:
		// Handled in FSM.Apply()

		default:
			glog.Errorf("Compaction: saw message of unknown type %v (log id %d, message id %v)\n", parsed.Type, i, parsed.Id)
			continue
		}
	}

	// We repeatedly compact, since the result of one compaction can affect the
	// result of other compactions (see compaction_test.go for examples).
	changed := true
	pass := 0
	for changed {
		log.Printf("Compaction pass %d\n", pass)
		pass++
		var err error
		changed, err = s.compact(db)
		if err != nil {
			return err
		}
	}

	var deleteSessions int
	if err := db.QueryRow("SELECT COUNT(*) FROM deleteSessionWin").Scan(&deleteSessions); err != nil {
		return err
	}
	log.Printf("%d deleteSessions left in compaction window\n", deleteSessions)
	numDeleteSessionsWin.Set(float64(deleteSessions))

	if _, err := db.Exec(`
		DROP VIEW allMessagesWin;
		DROP VIEW createSessionWin;
		DROP VIEW deleteSessionWin;
		`); err != nil {
		return err
	}

	for _, cmd := range ircserver.Commands {
		if cmd.CompactionDropViews == nil {
			continue
		}
		if err := cmd.CompactionDropViews(db); err != nil {
			return err
		}
	}

	if _, err := db.Exec("VACUUM"); err != nil {
		return err
	}

	log.Printf("Copying non-deleted messages into snapshot\n")

	snapshotBytes := 0
	available = iterator.First()
	for available {
		if err := iterator.Error(); err != nil {
			return err
		}
		i := binary.BigEndian.Uint64(iterator.Key())
		value := iterator.Value()
		if _, ok := s.del[i]; !ok {
			n, err := sink.Write(value)
			if err != nil {
				return err
			}
			snapshotBytes += n
		}
		available = iterator.Next()
	}
	log.Printf("snapshot: wrote %d bytes", snapshotBytes)

	if s.skipDeletionForCanary {
		log.Printf("Skipping deletion (canary mode)\n")
		return nil
	}

	log.Printf("Deleting compacted messages from disk\n")
	for idx, id := range s.del {
		if err := ircServer.Delete(id); err != nil {
			log.Panicf("Could not delete outputstream message: %v\n", err)
		}
		s.store.DeleteRange(idx, idx)
	}

	log.Printf("Snapshot done\n")
	compactionDurations.Observe(float64((time.Now().Sub(compactionStart)).Nanoseconds() / int64(time.Millisecond)))

	return nil
}

func (s *robustSnapshot) Release() {
}

// compact invokes all irc command’s |Compact| callbacks and runs the
// compaction for createSession and deleteSession messages.
func (s *robustSnapshot) compact(db *sql.DB) (bool, error) {
	const nonModifying = `
CREATE TABLE deleteIds AS
SELECT msgid FROM allMessagesWin WHERE irccommand = 'TOPIC' AND msgid NOT IN (SELECT msgid FROM paramsTopicWin)
UNION SELECT msgid FROM allMessagesWin WHERE irccommand = 'NICK' AND msgid NOT IN (SELECT msgid FROM paramsNickWin)
UNION SELECT msgid FROM allMessagesWin WHERE irccommand = 'USER' AND msgid NOT IN (SELECT msgid FROM paramsUserWin)
UNION SELECT msgid FROM allMessagesWin WHERE irccommand = 'JOIN' AND msgid NOT IN (SELECT msgid FROM paramsJoinWin)
UNION SELECT msgid FROM allMessagesWin WHERE irccommand = 'PART' AND msgid NOT IN (SELECT msgid FROM paramsPartWin)
UNION SELECT msgid FROM allMessagesWin WHERE irccommand = 'QUIT' AND msgid NOT IN (SELECT msgid FROM paramsQuitWin)
UNION SELECT msgid FROM allMessagesWin WHERE irccommand = 'MODE' AND msgid NOT IN (SELECT msgid FROM paramsModeWin)
UNION SELECT msgid FROM allMessagesWin WHERE irccommand = 'AWAY' AND msgid NOT IN (SELECT msgid FROM paramsAwayWin);
	`
	started := time.Now()
	if _, err := db.Exec(nonModifying); err != nil {
		return false, err
	}

	changed, err := s.markResultForCompaction(db, "non-modifying", started)
	if err != nil {
		return changed, err
	}

	if _, err := db.Exec("CREATE TABLE validCommands (command text not null)"); err != nil {
		return changed, err
	}
	validCommandInsert, err := db.Prepare("INSERT INTO validCommands (command) VALUES (?)")
	if err != nil {
		return changed, err
	}
	defer validCommandInsert.Close()

	immediatelyCompactable, err := db.Prepare("CREATE TABLE deleteIds AS SELECT msgid FROM allMessagesWin WHERE irccommand = ?")
	if err != nil {
		return changed, err
	}
	defer immediatelyCompactable.Close()

	for n, cmd := range ircserver.Commands {
		if _, err := validCommandInsert.Exec(n); err != nil {
			return changed, err
		}
		started := time.Now()
		if cmd.Compact != nil {
			if err := cmd.Compact(db); err != nil {
				return changed, err
			}
		} else if cmd.ImmediatelyCompactable {
			if _, err := immediatelyCompactable.Exec(n); err != nil {
				return changed, err
			}
		} else {
			continue
		}
		c, err := s.markResultForCompaction(db, "command "+n, started)
		if err != nil {
			return changed, err
		}
		changed = changed || c
	}

	const invalidCommands = `
	CREATE TABLE deleteIds AS SELECT msgid FROM allMessagesWin WHERE irccommand NOT IN (SELECT command FROM validCommands);
	DROP TABLE validCommands;
	`
	started = time.Now()
	_, err = db.Exec(invalidCommands)
	if err != nil {
		return changed, err
	}

	c, err := s.markResultForCompaction(db, "invalid commands", started)
	if err != nil {
		return changed, err
	}

	const stmt = `
-- Delete all deleteSession messages immediately preceded by a createSession
-- message (or commands in between which do not modify state of anything but
-- the session in question).
CREATE TABLE candidates AS
SELECT
    a.msgid AS msgid,
    a.session AS session
FROM
    (
        SELECT
            d.msgid AS msgid,
            d.session AS session,
            MAX(a.msgid) AS next_msgid
        FROM
            (SELECT msgid, session FROM deleteSessionWin
             UNION SELECT msgid, session FROM paramsQuitWin) AS d
            INNER JOIN allMessagesWin AS a
            ON (
                d.session = a.session AND
                (a.irccommand IS NULL OR
                 (a.irccommand != 'NICK' AND
                  a.irccommand != 'USER' AND
                  a.irccommand != 'PASS' AND
                  a.irccommand != 'QUIT')) AND
                a.msgid < d.msgid
            )
        GROUP BY d.msgid
    ) AS a
    INNER JOIN createSessionWin AS c
    ON (
        a.session = c.session AND
        a.next_msgid = c.msgid
    );
DELETE FROM deleteSession WHERE msgid IN (SELECT msgid FROM candidates);

INSERT INTO candidates
SELECT
    a.msgid,
    a.session
FROM
    allMessagesWin AS a
    LEFT JOIN candidates AS d
    ON (
        a.session = d.session
    )
    WHERE d.session IS NOT NULL;

CREATE TABLE deleteIds AS SELECT msgid FROM candidates;
DROP TABLE candidates;
DELETE FROM createSession WHERE msgid IN (SELECT msgid FROM deleteIds);
`

	started = time.Now()
	_, err = db.Exec(stmt)
	if err != nil {
		return changed, err
	}

	c, err = s.markResultForCompaction(db, "deleteSession, createSession", started)
	if err != nil {
		return changed, err
	}
	return changed || c, nil
}

// markResultForCompaction updates |s.del| with all messages whose ids are in
// the deleteIds table. It then removes these messages from the allMessages
// table and drops the deleteIds table.
func (s *robustSnapshot) markResultForCompaction(db *sql.DB, label string, started time.Time) (bool, error) {
	changed := false
	rows, err := db.Query("SELECT msgid FROM deleteIds")
	if err != nil {
		return changed, err
	}
	defer rows.Close()
	deleted := 0
	for rows.Next() {
		var msgid int64
		if err := rows.Scan(&msgid); err != nil {
			return changed, err
		}
		idx, ok := s.idToIdx[msgid]
		if !ok {
			return changed, fmt.Errorf("msgid=%v not in idToIdx", msgid)
		}
		s.del[idx] = types.RobustId{Id: msgid}
		changed = true
		deleted++
	}
	if err := rows.Err(); err != nil {
		return changed, err
	}
	log.Printf("%d rows deleted in %v (step %q)\n", deleted, time.Since(started), label)

	if _, err := db.Exec("DELETE FROM allMessages WHERE msgid IN (SELECT msgid FROM deleteIds)"); err != nil {
		return changed, err
	}

	_, err = db.Exec("DROP TABLE deleteIds")
	return changed, nil
}
<|endoftext|>


<commit_before>package d2x

import "math/big"

type NefIFD0 struct {
	NewSubfileType            uint32     `tiff:"field,tag=254"`
	ImageWidth                uint32     `tiff:"field,tag=256"`
	ImageLength               uint32     `tiff:"field,tag=257"`
	BitsPerSample             []uint16   `tiff:"field,tag=258"`
	Compression               uint16     `tiff:"field,tag=259"`
	PhotometricInterpretation uint16     `tiff:"field,tag=262"`
	Make                      string     `tiff:"field,tag=271"`
	Model                     string     `tiff:"field,tag=272"`
	StripOffsets              uint32     `tiff:"field,tag=273"`
	Orientation               uint16     `tiff:"field,tag=274"`
	SamplesPerPixel           uint16     `tiff:"field,tag=277"`
	RowsPerStrip              uint32     `tiff:"field,tag=278"`
	StripByteCounts           uint32     `tiff:"field,tag=279"`
	XResolution               *big.Rat   `tiff:"field,tag=282"`
	YResolution               *big.Rat   `tiff:"field,tag=283"`
	PlanarConfiguration       uint16     `tiff:"field,tag=284"`
	ResolutionUnit            uint16     `tiff:"field,tag=296"`
	Software                  string     `tiff:"field,tag=305"`
	DateTime                  string     `tiff:"field,tag=306"`
	SubIFDs                   []uint32   `tiff:"field,tag=330"`
	ReferenceBlackWhite       []*big.Rat `tiff:"field,tag=532"`
	ExifIFD                   uint32     `tiff:"field,tag=34665"`
	GPSIFD                    uint32     `tiff:"field,tag=34853"`
	DateTimeOriginal          string     `tiff:"field,tag=36867"`
	TIFFEPStandardID          []byte     `tiff:"field,tag=37398"`
}

type NefIFD0SubIFD0 struct {
	NewSubfileType              uint32   `tiff:"field,tag=254"`
	Compression                 uint16   `tiff:"field,tag=259"`
	XResolution                 *big.Rat `tiff:"field,tag=282"`
	YResolution                 *big.Rat `tiff:"field,tag=283"`
	ResolutionUnit              uint16   `tiff:"field,tag=296"`
	JPEGInterchangeFormat       uint32   `tiff:"field,tag=513"`
	JPEGInterchangeFormatLength uint32   `tiff:"field,tag=514"`
	YCbCrPositioning            uint16   `tiff:"field,tag=531"`
}

type NefIFD0SubIFD1 struct {
	NewSubfileType            uint32   `tiff:"field,tag=254"`
	ImageWidth                uint32   `tiff:"field,tag=256"`
	ImageLength               uint32   `tiff:"field,tag=257"`
	BitsPerSample             uint16   `tiff:"field,tag=258"`
	Compression               uint16   `tiff:"field,tag=259"`
	PhotometricInterpretation uint16   `tiff:"field,tag=262"`
	StripOffsets              []uint32 `tiff:"field,tag=273"`
	SamplesPerPixel           uint16   `tiff:"field,tag=277"`
	RowsPerStrip              uint32   `tiff:"field,tag=278"`
	StripByteCounts           []uint32 `tiff:"field,tag=279"`
	XResolution               *big.Rat `tiff:"field,tag=282"`
	YResolution               *big.Rat `tiff:"field,tag=283"`
	PlanarConfiguration       uint16   `tiff:"field,tag=284"`
	ResolutionUnit            uint16   `tiff:"field,tag=296"`
	CFARepeatPatternDim       []uint16 `tiff:"field,tag=33421"`
	CFAPattern                []byte   `tiff:"field,tag=33422"`
	SensingMethod             uint16   `tiff:"field,tag=37399"`
}

type NefIFD0ExifIFD struct {
	ExposureTime          *big.Rat `tiff:"field,tag=33434"`
	FNumber               *big.Rat `tiff:"field,tag=33437"`
	ExposureProgram       uint16   `tiff:"field,tag=34850"`
	ISOSpeedRatings       uint16   `tiff:"field,tag=34855"`
	DateTimeOriginal      string   `tiff:"field,tag=36867"`
	DateTimeDigitized     string   `tiff:"field,tag=36868"`
	ExposureBiasValue     *big.Rat `tiff:"field,tag=37380"`
	MaxApertureValue      *big.Rat `tiff:"field,tag=37381"`
	MeteringMode          uint16   `tiff:"field,tag=37383"`
	LightSource           uint16   `tiff:"field,tag=37384"`
	Flash                 uint16   `tiff:"field,tag=37385"`
	FocalLength           *big.Rat `tiff:"field,tag=37386"`
	MakerNote             []byte   `tiff:"field,tag=37500"`
	UserComment           []byte   `tiff:"field,tag=37510"`
	SubsecTime            string   `tiff:"field,tag=37520"`
	SubsecTimeOriginal    string   `tiff:"field,tag=37521"`
	SubsecTimeDigitized   string   `tiff:"field,tag=37522"`
	SensingMethod         uint16   `tiff:"field,tag=41495"`
	FileSource            byte     `tiff:"field,tag=41728"`
	SceneType             byte     `tiff:"field,tag=41729"`
	CFAPattern            []byte   `tiff:"field,tag=41730"`
	CustomRendered        uint16   `tiff:"field,tag=41985"`
	ExposureMode          uint16   `tiff:"field,tag=41986"`
	WhiteBalance          uint16   `tiff:"field,tag=41987"`
	DigitalZoomRatio      *big.Rat `tiff:"field,tag=41988"`
	FocalLengthIn35mmFilm uint16   `tiff:"field,tag=41989"`
	SceneCaptureType      uint16   `tiff:"field,tag=41990"`
	GainControl           uint16   `tiff:"field,tag=41991"`
	Contrast              uint16   `tiff:"field,tag=41992"`
	Saturation            uint16   `tiff:"field,tag=41993"`
	Sharpness             uint16   `tiff:"field,tag=41994"`
	SubjectDistanceRange  uint16   `tiff:"field,tag=41996"`
}

type NefIFD0GPSIFD struct {
	GPSVersionID []byte `tiff:"field,tag=0"`
}
<commit_msg>nikon/d2x: Add Makernote type<commit_after>package d2x

import "math/big"

type NefIFD0 struct {
	NewSubfileType            uint32     `tiff:"field,tag=254"`
	ImageWidth                uint32     `tiff:"field,tag=256"`
	ImageLength               uint32     `tiff:"field,tag=257"`
	BitsPerSample             []uint16   `tiff:"field,tag=258"`
	Compression               uint16     `tiff:"field,tag=259"`
	PhotometricInterpretation uint16     `tiff:"field,tag=262"`
	Make                      string     `tiff:"field,tag=271"`
	Model                     string     `tiff:"field,tag=272"`
	StripOffsets              uint32     `tiff:"field,tag=273"`
	Orientation               uint16     `tiff:"field,tag=274"`
	SamplesPerPixel           uint16     `tiff:"field,tag=277"`
	RowsPerStrip              uint32     `tiff:"field,tag=278"`
	StripByteCounts           uint32     `tiff:"field,tag=279"`
	XResolution               *big.Rat   `tiff:"field,tag=282"`
	YResolution               *big.Rat   `tiff:"field,tag=283"`
	PlanarConfiguration       uint16     `tiff:"field,tag=284"`
	ResolutionUnit            uint16     `tiff:"field,tag=296"`
	Software                  string     `tiff:"field,tag=305"`
	DateTime                  string     `tiff:"field,tag=306"`
	SubIFDs                   []uint32   `tiff:"field,tag=330"`
	ReferenceBlackWhite       []*big.Rat `tiff:"field,tag=532"`
	ExifIFD                   uint32     `tiff:"field,tag=34665"`
	GPSIFD                    uint32     `tiff:"field,tag=34853"`
	DateTimeOriginal          string     `tiff:"field,tag=36867"`
	TIFFEPStandardID          []byte     `tiff:"field,tag=37398"`
}

type NefIFD0SubIFD0 struct {
	NewSubfileType              uint32   `tiff:"field,tag=254"`
	Compression                 uint16   `tiff:"field,tag=259"`
	XResolution                 *big.Rat `tiff:"field,tag=282"`
	YResolution                 *big.Rat `tiff:"field,tag=283"`
	ResolutionUnit              uint16   `tiff:"field,tag=296"`
	JPEGInterchangeFormat       uint32   `tiff:"field,tag=513"`
	JPEGInterchangeFormatLength uint32   `tiff:"field,tag=514"`
	YCbCrPositioning            uint16   `tiff:"field,tag=531"`
}

type NefIFD0SubIFD1 struct {
	NewSubfileType            uint32   `tiff:"field,tag=254"`
	ImageWidth                uint32   `tiff:"field,tag=256"`
	ImageLength               uint32   `tiff:"field,tag=257"`
	BitsPerSample             uint16   `tiff:"field,tag=258"`
	Compression               uint16   `tiff:"field,tag=259"`
	PhotometricInterpretation uint16   `tiff:"field,tag=262"`
	StripOffsets              []uint32 `tiff:"field,tag=273"`
	SamplesPerPixel           uint16   `tiff:"field,tag=277"`
	RowsPerStrip              uint32   `tiff:"field,tag=278"`
	StripByteCounts           []uint32 `tiff:"field,tag=279"`
	XResolution               *big.Rat `tiff:"field,tag=282"`
	YResolution               *big.Rat `tiff:"field,tag=283"`
	PlanarConfiguration       uint16   `tiff:"field,tag=284"`
	ResolutionUnit            uint16   `tiff:"field,tag=296"`
	CFARepeatPatternDim       []uint16 `tiff:"field,tag=33421"`
	CFAPattern                []byte   `tiff:"field,tag=33422"`
	SensingMethod             uint16   `tiff:"field,tag=37399"`
}

type NefIFD0ExifIFD struct {
	ExposureTime          *big.Rat `tiff:"field,tag=33434"`
	FNumber               *big.Rat `tiff:"field,tag=33437"`
	ExposureProgram       uint16   `tiff:"field,tag=34850"`
	ISOSpeedRatings       uint16   `tiff:"field,tag=34855"`
	DateTimeOriginal      string   `tiff:"field,tag=36867"`
	DateTimeDigitized     string   `tiff:"field,tag=36868"`
	ExposureBiasValue     *big.Rat `tiff:"field,tag=37380"`
	MaxApertureValue      *big.Rat `tiff:"field,tag=37381"`
	MeteringMode          uint16   `tiff:"field,tag=37383"`
	LightSource           uint16   `tiff:"field,tag=37384"`
	Flash                 uint16   `tiff:"field,tag=37385"`
	FocalLength           *big.Rat `tiff:"field,tag=37386"`
	MakerNote             []byte   `tiff:"field,tag=37500"`
	UserComment           []byte   `tiff:"field,tag=37510"`
	SubsecTime            string   `tiff:"field,tag=37520"`
	SubsecTimeOriginal    string   `tiff:"field,tag=37521"`
	SubsecTimeDigitized   string   `tiff:"field,tag=37522"`
	SensingMethod         uint16   `tiff:"field,tag=41495"`
	FileSource            byte     `tiff:"field,tag=41728"`
	SceneType             byte     `tiff:"field,tag=41729"`
	CFAPattern            []byte   `tiff:"field,tag=41730"`
	CustomRendered        uint16   `tiff:"field,tag=41985"`
	ExposureMode          uint16   `tiff:"field,tag=41986"`
	WhiteBalance          uint16   `tiff:"field,tag=41987"`
	DigitalZoomRatio      *big.Rat `tiff:"field,tag=41988"`
	FocalLengthIn35mmFilm uint16   `tiff:"field,tag=41989"`
	SceneCaptureType      uint16   `tiff:"field,tag=41990"`
	GainControl           uint16   `tiff:"field,tag=41991"`
	Contrast              uint16   `tiff:"field,tag=41992"`
	Saturation            uint16   `tiff:"field,tag=41993"`
	Sharpness             uint16   `tiff:"field,tag=41994"`
	SubjectDistanceRange  uint16   `tiff:"field,tag=41996"`
}

type NefIFD0GPSIFD struct {
	GPSVersionID []byte `tiff:"field,tag=0"`
}

type Makernote struct {
	Version                   []byte     `tiff:"field,tag=1"`
	ISOSpeed                  []uint16   `tiff:"field,tag=2"`
	Quality                   string     `tiff:"field,tag=4"`
	WhiteBalance              string     `tiff:"field,tag=5"`
	Sharpening                string     `tiff:"field,tag=6"`
	FocusMode                 string     `tiff:"field,tag=7"`
	FlashSetting              string     `tiff:"field,tag=8"`
	FlashDevice               string     `tiff:"field,tag=9"`
	WhiteBalanceFineTune      int16      `tiff:"field,tag=11"`
	ProgramShift              []byte     `tiff:"field,tag=13"`
	ExposureDifference        []byte     `tiff:"field,tag=14"`
	PreviewIFD                uint32     `tiff:"field,tag=17"`
	ISOSetting                []uint16   `tiff:"field,tag=19"`
	ExternalFlashExposureComp []byte     `tiff:"field,tag=23"`
	FlashExposureBracketValue []byte     `tiff:"field,tag=24"`
	ExposureBracketValue      *big.Rat   `tiff:"field,tag=25"`
	CropHiSpeed               []uint16   `tiff:"field,tag=27"`
	ExposureTuning            []byte     `tiff:"field,tag=28"`
	SerialNumber              string     `tiff:"field,tag=29"`
	ColorSpace                uint16     `tiff:"field,tag=30"`
	ToneComp                  string     `tiff:"field,tag=129"`
	LensType                  byte       `tiff:"field,tag=131"`
	Lens                      []*big.Rat `tiff:"field,tag=132"`
	FlashMode                 byte       `tiff:"field,tag=135"`
	AFInfo                    []byte     `tiff:"field,tag=136"`
	ShootingMode              uint16     `tiff:"field,tag=137"`
	AutoBracketRelease        uint16     `tiff:"field,tag=138"`
	LensFStops                []byte     `tiff:"field,tag=139"`
	ContrastCurve             []byte     `tiff:"field,tag=140"`
	ColorHue                  string     `tiff:"field,tag=141"`
	LightSource               string     `tiff:"field,tag=144"`
	ShotInfo                  []byte     `tiff:"field,tag=145"`
	HueAdjustment             int16      `tiff:"field,tag=146"`
	NoiseReduction            string     `tiff:"field,tag=149"`
	ColorBalance              []byte     `tiff:"field,tag=151"`
	LensData                  []byte     `tiff:"field,tag=152"`
	RawImageCenter            []uint16   `tiff:"field,tag=153"`
	SensorPixelSize           []*big.Rat `tiff:"field,tag=154"`
	UNKNOWN_TAG_163           byte       `tiff:"field,tag=163"`
	UNKNOWN_TAG_164           []byte     `tiff:"field,tag=164"`
	ImageCount                uint32     `tiff:"field,tag=165"`
	DeletedImageCount         uint32     `tiff:"field,tag=166"`
	ShutterCount              uint32     `tiff:"field,tag=167"`
	FlashInfo                 []byte     `tiff:"field,tag=168"`
	Saturation                string     `tiff:"field,tag=170"`
	MultiExposure             []byte     `tiff:"field,tag=176"`
	HighISONoiseReduction     uint16     `tiff:"field,tag=177"`
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/base64"
	"encoding/json"
)

// Cluster represents high-level information about a LXD cluster.
//
// swagger:model
//
// API extension: clustering
type Cluster struct {
	// Name of the cluster member answering the request
	// Example: lxd01
	ServerName string `json:"server_name" yaml:"server_name"`

	// Whether clustering is enabled
	// Example: true
	Enabled bool `json:"enabled" yaml:"enabled"`

	// List of member configuration keys (used during join)
	// Example: []
	//
	// API extension: clustering_join
	MemberConfig []ClusterMemberConfigKey `json:"member_config" yaml:"member_config"`
}

// ClusterMemberConfigKey represents a single config key that a new member of
// the cluster is required to provide when joining.
//
// The Value field is empty when getting clustering information with GET
// /1.0/cluster, and should be filled by the joining node when performing a PUT
// /1.0/cluster join request.
//
// swagger:model
//
// API extension: clustering_join
type ClusterMemberConfigKey struct {
	// The kind of configuration key (network, storage-pool, ...)
	// Example: storage-pool
	Entity string `json:"entity" yaml:"entity"`

	// The name of the object requiring this key
	// Example: local
	Name string `json:"name" yaml:"name"`

	// The name of the key
	// Example: source
	Key string `json:"key" yaml:"key"`

	// The value on the answering cluster member
	// Example: /dev/sdb
	Value string `json:"value" yaml:"value"`

	// A human friendly description key
	// Example: "source" property for storage pool "local"
	Description string `json:"description" yaml:"description"`
}

// ClusterPut represents the fields required to bootstrap or join a LXD
// cluster.
//
// swagger:model
//
// API extension: clustering
type ClusterPut struct {
	Cluster `yaml:",inline"`

	// The address of the cluster you wish to join
	// Example: 10.0.0.1:8443
	ClusterAddress string `json:"cluster_address" yaml:"cluster_address"`

	// The expected certificate (X509 PEM encoded) for the cluster
	// Example: X509 PEM certificate
	ClusterCertificate string `json:"cluster_certificate" yaml:"cluster_certificate"`

	// The local address to use for cluster communication
	// Example: 10.0.0.2:8443
	//
	// API extension: clustering_join
	ServerAddress string `json:"server_address" yaml:"server_address"`

	// The trust password of the cluster you're trying to join
	// Example: blah
	//
	// API extension: clustering_join
	ClusterPassword string `json:"cluster_password" yaml:"cluster_password"`
}

// ClusterMembersPost represents the fields required to request a join token to add a member to the cluster.
//
// swagger:model
//
// API extension: clustering_join_token
type ClusterMembersPost struct {
	// The name of the new cluster member
	// Example: lxd02
	ServerName string `json:"server_name" yaml:"server_name"`
}

// ClusterMemberJoinToken represents the fields contained within an encoded cluster member join token.
//
// swagger:model
//
// API extension: clustering_join_token
type ClusterMemberJoinToken struct {
	// The name of the new cluster member
	// Example: lxd02
	ServerName string `json:"server_name" yaml:"server_name"`

	// The fingerprint of the network certificate
	// Example: 57bb0ff4340b5bb28517e062023101adf788c37846dc8b619eb2c3cb4ef29436
	Fingerprint string `json:"fingerprint" yaml:"fingerprint"`

	// The addresses of existing online cluster members
	// Example: ["10.98.30.229:8443"]
	Addresses []string `json:"addresses" yaml:"addresses"`

	// The random join secret.
	// Example: 2b2284d44db32675923fe0d2020477e0e9be11801ff70c435e032b97028c35cd
	Secret string `json:"secret" yaml:"secret"`
}

// String encodes the cluster member join token as JSON and then base64.
func (t *ClusterMemberJoinToken) String() string {
	joinTokenJSON, err := json.Marshal(t)
	if err != nil {
		return ""
	}

	return base64.StdEncoding.EncodeToString(joinTokenJSON)
}

// ClusterMemberPost represents the fields required to rename a LXD node.
//
// swagger:model
//
// API extension: clustering
type ClusterMemberPost struct {
	// The new name of the cluster member
	// Example: lxd02
	ServerName string `json:"server_name" yaml:"server_name"`
}

// ClusterMember represents the a LXD node in the cluster.
//
// swagger:model
//
// API extension: clustering
type ClusterMember struct {
	ClusterMemberPut `yaml:",inline"`

	// Name of the cluster member
	// Example: lxd01
	ServerName string `json:"server_name" yaml:"server_name"`

	// URL at which the cluster member can be reached
	// Example: https://10.0.0.1:8443
	URL string `json:"url" yaml:"url"`

	// Whether the cluster member is a database server
	// Example: true
	Database bool `json:"database" yaml:"database"`

	// Current status
	// Example: Online
	Status string `json:"status" yaml:"status"`

	// Additional status information
	// Example: fully operational
	Message string `json:"message" yaml:"message"`

	// The primary architecture of the cluster member
	// Example: x86_64
	//
	// API extension: clustering_architecture
	Architecture string `json:"architecture" yaml:"architecture"`
}

// Writable converts a full Profile struct into a ProfilePut struct (filters read-only fields)
func (member *ClusterMember) Writable() ClusterMemberPut {
	return member.ClusterMemberPut
}

// ClusterMemberPut represents the the modifiable fields of a LXD cluster member
//
// swagger:model
//
// API extension: clustering_edit_roles
type ClusterMemberPut struct {
	// List of roles held by this cluster member
	// Example: ["database"]
	//
	// API extension: clustering_roles
	Roles []string `json:"roles" yaml:"roles"`

	// Name of the failure domain for this cluster member
	// Example: rack1
	//
	// API extension: clustering_failure_domains
	FailureDomain string `json:"failure_domain" yaml:"failure_domain"`

	// Cluster member description
	// Example: AMD Epyc 32c/64t
	//
	// API extension: clustering_description
	Description string `json:"description" yaml:"description"`

	// Additional configuration information
	// Example: {"scheduler.instance": "all"}
	//
	// API extension: clustering_config
	Config map[string]string `json:"config" yaml:"config"`

	// List of cluster groups this member belongs to
	// Example: ["group1", "group2"]
	//
	// API extension: clustering_groups
	Groups []string `json:"groups" yaml:"groups"`
}

// ClusterCertificatePut represents the certificate and key pair for all members in a LXD Cluster
//
// swagger:model
//
// API extension: clustering_update_certs
type ClusterCertificatePut struct {
	// The new certificate (X509 PEM encoded) for the cluster
	// Example: X509 PEM certificate
	ClusterCertificate string `json:"cluster_certificate" yaml:"cluster_certificate"`

	// The new certificate key (X509 PEM encoded) for the cluster
	// Example: X509 PEM certificate key
	ClusterCertificateKey string `json:"cluster_certificate_key" yaml:"cluster_certificate_key"`
}

// ClusterMemberStatePost represents the fields required to evacuate a cluster member.
//
// swagger:model
//
// API extension: clustering_evacuation
type ClusterMemberStatePost struct {
	// The action to be performed. Valid actions are "evacuate" and "restore".
	// Example: evacuate
	Action string `json:"action" yaml:"action"`
}

// ClusterGroupsPost represents the fields available for a new cluster group.
//
// swagger:model
//
// API extension: clustering_groups
type ClusterGroupsPost struct {
	ClusterGroupPut

	// The new name of the cluster group
	// Example: group1
	Name string `json:"name" yaml:"name"`
}

// ClusterGroup represents a cluster group.
//
// swagger:model
//
// API extension: clustering_groups
type ClusterGroup struct {
	ClusterGroupPut  `yaml:",inline"`
	ClusterGroupPost `yaml:",inline"`
}

// ClusterGroupPost represents the fields required to rename a cluster group.
//
// swagger:model
//
// API extension: clustering_groups
type ClusterGroupPost struct {
	// The new name of the cluster group
	// Example: group1
	Name string `json:"name" yaml:"name"`
}

// ClusterGroupPut represents the modifiable fields of a cluster group.
//
// swagger:model
//
// API extension: clustering_groups
type ClusterGroupPut struct {
	// The description of the cluster group
	// Example: amd64 servers
	Description string `json:"description" yaml:"description"`

	// List of members in this group
	// Example: ["node1", "node3"]
	Members []string `json:"members" yaml:"members"`
}

// Writable converts a full ClusterGroup struct into a ClusterGroupPut struct (filters read-only fields)
func (c *ClusterGroup) Writable() ClusterGroupPut {
	return c.ClusterGroupPut
}
<commit_msg>shared/api: Add Mode to ClusterMemberStatePost<commit_after>package api

import (
	"encoding/base64"
	"encoding/json"
)

// Cluster represents high-level information about a LXD cluster.
//
// swagger:model
//
// API extension: clustering
type Cluster struct {
	// Name of the cluster member answering the request
	// Example: lxd01
	ServerName string `json:"server_name" yaml:"server_name"`

	// Whether clustering is enabled
	// Example: true
	Enabled bool `json:"enabled" yaml:"enabled"`

	// List of member configuration keys (used during join)
	// Example: []
	//
	// API extension: clustering_join
	MemberConfig []ClusterMemberConfigKey `json:"member_config" yaml:"member_config"`
}

// ClusterMemberConfigKey represents a single config key that a new member of
// the cluster is required to provide when joining.
//
// The Value field is empty when getting clustering information with GET
// /1.0/cluster, and should be filled by the joining node when performing a PUT
// /1.0/cluster join request.
//
// swagger:model
//
// API extension: clustering_join
type ClusterMemberConfigKey struct {
	// The kind of configuration key (network, storage-pool, ...)
	// Example: storage-pool
	Entity string `json:"entity" yaml:"entity"`

	// The name of the object requiring this key
	// Example: local
	Name string `json:"name" yaml:"name"`

	// The name of the key
	// Example: source
	Key string `json:"key" yaml:"key"`

	// The value on the answering cluster member
	// Example: /dev/sdb
	Value string `json:"value" yaml:"value"`

	// A human friendly description key
	// Example: "source" property for storage pool "local"
	Description string `json:"description" yaml:"description"`
}

// ClusterPut represents the fields required to bootstrap or join a LXD
// cluster.
//
// swagger:model
//
// API extension: clustering
type ClusterPut struct {
	Cluster `yaml:",inline"`

	// The address of the cluster you wish to join
	// Example: 10.0.0.1:8443
	ClusterAddress string `json:"cluster_address" yaml:"cluster_address"`

	// The expected certificate (X509 PEM encoded) for the cluster
	// Example: X509 PEM certificate
	ClusterCertificate string `json:"cluster_certificate" yaml:"cluster_certificate"`

	// The local address to use for cluster communication
	// Example: 10.0.0.2:8443
	//
	// API extension: clustering_join
	ServerAddress string `json:"server_address" yaml:"server_address"`

	// The trust password of the cluster you're trying to join
	// Example: blah
	//
	// API extension: clustering_join
	ClusterPassword string `json:"cluster_password" yaml:"cluster_password"`
}

// ClusterMembersPost represents the fields required to request a join token to add a member to the cluster.
//
// swagger:model
//
// API extension: clustering_join_token
type ClusterMembersPost struct {
	// The name of the new cluster member
	// Example: lxd02
	ServerName string `json:"server_name" yaml:"server_name"`
}

// ClusterMemberJoinToken represents the fields contained within an encoded cluster member join token.
//
// swagger:model
//
// API extension: clustering_join_token
type ClusterMemberJoinToken struct {
	// The name of the new cluster member
	// Example: lxd02
	ServerName string `json:"server_name" yaml:"server_name"`

	// The fingerprint of the network certificate
	// Example: 57bb0ff4340b5bb28517e062023101adf788c37846dc8b619eb2c3cb4ef29436
	Fingerprint string `json:"fingerprint" yaml:"fingerprint"`

	// The addresses of existing online cluster members
	// Example: ["10.98.30.229:8443"]
	Addresses []string `json:"addresses" yaml:"addresses"`

	// The random join secret.
	// Example: 2b2284d44db32675923fe0d2020477e0e9be11801ff70c435e032b97028c35cd
	Secret string `json:"secret" yaml:"secret"`
}

// String encodes the cluster member join token as JSON and then base64.
func (t *ClusterMemberJoinToken) String() string {
	joinTokenJSON, err := json.Marshal(t)
	if err != nil {
		return ""
	}

	return base64.StdEncoding.EncodeToString(joinTokenJSON)
}

// ClusterMemberPost represents the fields required to rename a LXD node.
//
// swagger:model
//
// API extension: clustering
type ClusterMemberPost struct {
	// The new name of the cluster member
	// Example: lxd02
	ServerName string `json:"server_name" yaml:"server_name"`
}

// ClusterMember represents the a LXD node in the cluster.
//
// swagger:model
//
// API extension: clustering
type ClusterMember struct {
	ClusterMemberPut `yaml:",inline"`

	// Name of the cluster member
	// Example: lxd01
	ServerName string `json:"server_name" yaml:"server_name"`

	// URL at which the cluster member can be reached
	// Example: https://10.0.0.1:8443
	URL string `json:"url" yaml:"url"`

	// Whether the cluster member is a database server
	// Example: true
	Database bool `json:"database" yaml:"database"`

	// Current status
	// Example: Online
	Status string `json:"status" yaml:"status"`

	// Additional status information
	// Example: fully operational
	Message string `json:"message" yaml:"message"`

	// The primary architecture of the cluster member
	// Example: x86_64
	//
	// API extension: clustering_architecture
	Architecture string `json:"architecture" yaml:"architecture"`
}

// Writable converts a full Profile struct into a ProfilePut struct (filters read-only fields)
func (member *ClusterMember) Writable() ClusterMemberPut {
	return member.ClusterMemberPut
}

// ClusterMemberPut represents the the modifiable fields of a LXD cluster member
//
// swagger:model
//
// API extension: clustering_edit_roles
type ClusterMemberPut struct {
	// List of roles held by this cluster member
	// Example: ["database"]
	//
	// API extension: clustering_roles
	Roles []string `json:"roles" yaml:"roles"`

	// Name of the failure domain for this cluster member
	// Example: rack1
	//
	// API extension: clustering_failure_domains
	FailureDomain string `json:"failure_domain" yaml:"failure_domain"`

	// Cluster member description
	// Example: AMD Epyc 32c/64t
	//
	// API extension: clustering_description
	Description string `json:"description" yaml:"description"`

	// Additional configuration information
	// Example: {"scheduler.instance": "all"}
	//
	// API extension: clustering_config
	Config map[string]string `json:"config" yaml:"config"`

	// List of cluster groups this member belongs to
	// Example: ["group1", "group2"]
	//
	// API extension: clustering_groups
	Groups []string `json:"groups" yaml:"groups"`
}

// ClusterCertificatePut represents the certificate and key pair for all members in a LXD Cluster
//
// swagger:model
//
// API extension: clustering_update_certs
type ClusterCertificatePut struct {
	// The new certificate (X509 PEM encoded) for the cluster
	// Example: X509 PEM certificate
	ClusterCertificate string `json:"cluster_certificate" yaml:"cluster_certificate"`

	// The new certificate key (X509 PEM encoded) for the cluster
	// Example: X509 PEM certificate key
	ClusterCertificateKey string `json:"cluster_certificate_key" yaml:"cluster_certificate_key"`
}

// ClusterMemberStatePost represents the fields required to evacuate a cluster member.
//
// swagger:model
//
// API extension: clustering_evacuation
type ClusterMemberStatePost struct {
	// The action to be performed. Valid actions are "evacuate" and "restore".
	// Example: evacuate
	Action string `json:"action" yaml:"action"`

	// Override the configured evacuation mode.
	// Example: stop
	//
	// API extension: clustering_evacuate_mode
	Mode string `json:"mode" yaml:"mode"`
}

// ClusterGroupsPost represents the fields available for a new cluster group.
//
// swagger:model
//
// API extension: clustering_groups
type ClusterGroupsPost struct {
	ClusterGroupPut

	// The new name of the cluster group
	// Example: group1
	Name string `json:"name" yaml:"name"`
}

// ClusterGroup represents a cluster group.
//
// swagger:model
//
// API extension: clustering_groups
type ClusterGroup struct {
	ClusterGroupPut  `yaml:",inline"`
	ClusterGroupPost `yaml:",inline"`
}

// ClusterGroupPost represents the fields required to rename a cluster group.
//
// swagger:model
//
// API extension: clustering_groups
type ClusterGroupPost struct {
	// The new name of the cluster group
	// Example: group1
	Name string `json:"name" yaml:"name"`
}

// ClusterGroupPut represents the modifiable fields of a cluster group.
//
// swagger:model
//
// API extension: clustering_groups
type ClusterGroupPut struct {
	// The description of the cluster group
	// Example: amd64 servers
	Description string `json:"description" yaml:"description"`

	// List of members in this group
	// Example: ["node1", "node3"]
	Members []string `json:"members" yaml:"members"`
}

// Writable converts a full ClusterGroup struct into a ClusterGroupPut struct (filters read-only fields)
func (c *ClusterGroup) Writable() ClusterGroupPut {
	return c.ClusterGroupPut
}
<|endoftext|>


<commit_before>package api

// NetworksPost represents the fields of a new LXD network
//
// API extension: network
type NetworksPost struct {
	NetworkPut `yaml:",inline"`

	Name string `json:"name" yaml:"name"`
	Type string `json:"type" yaml:"type"`
}

// NetworkPost represents the fields required to rename a LXD network
//
// API extension: network
type NetworkPost struct {
	Name string `json:"name" yaml:"name"`
}

// NetworkPut represents the modifiable fields of a LXD network
//
// API extension: network
type NetworkPut struct {
	Config map[string]string `json:"config" yaml:"config"`

	// API extension: entity_description
	Description string `json:"description" yaml:"description"`
}

// NetworkStatusPending network is pending creation on other cluster nodes.
const NetworkStatusPending = "Pending"

// NetworkStatusCreated network is fully created.
const NetworkStatusCreated = "Created"

// NetworkStatusErrored network is in error status.
const NetworkStatusErrored = "Errored"

// NetworkStatusUnknown network is in unknown status.
const NetworkStatusUnknown = "Unknown"

// Network represents a LXD network
type Network struct {
	NetworkPut `yaml:",inline"`

	Name   string   `json:"name" yaml:"name"`
	Type   string   `json:"type" yaml:"type"`
	UsedBy []string `json:"used_by" yaml:"used_by"`

	// API extension: network
	Managed bool `json:"managed" yaml:"managed"`

	// API extension: clustering
	Status    string   `json:"status" yaml:"status"`
	Locations []string `json:"locations" yaml:"locations"`
}

// Writable converts a full Network struct into a NetworkPut struct (filters read-only fields)
func (network *Network) Writable() NetworkPut {
	return network.NetworkPut
}

// NetworkLease represents a DHCP lease
//
// API extension: network_leases
type NetworkLease struct {
	Hostname string `json:"hostname" yaml:"hostname"`
	Hwaddr   string `json:"hwaddr" yaml:"hwaddr"`
	Address  string `json:"address" yaml:"address"`
	Type     string `json:"type" yaml:"type"`

	// API extension: network_leases_location
	Location string `json:"location" yaml:"location"`
}

// NetworkState represents the network state
type NetworkState struct {
	Addresses []NetworkStateAddress `json:"addresses" yaml:"addresses"`
	Counters  NetworkStateCounters  `json:"counters" yaml:"counters"`
	Hwaddr    string                `json:"hwaddr" yaml:"hwaddr"`
	Mtu       int                   `json:"mtu" yaml:"mtu"`
	State     string                `json:"state" yaml:"state"`
	Type      string                `json:"type" yaml:"type"`

	// API extension: network_state_bond_bridge
	Bond   *NetworkStateBond   `json:"bond" yaml:"bond"`
	Bridge *NetworkStateBridge `json:"bridge" yaml:"bridge"`

	// API extension: network_state_vlan
	VLAN *NetworkStateVLAN `json:"vlan" yaml:"vlan"`
}

// NetworkStateAddress represents a network address
type NetworkStateAddress struct {
	Family  string `json:"family" yaml:"family"`
	Address string `json:"address" yaml:"address"`
	Netmask string `json:"netmask" yaml:"netmask"`
	Scope   string `json:"scope" yaml:"scope"`
}

// NetworkStateCounters represents packet counters
type NetworkStateCounters struct {
	BytesReceived   int64 `json:"bytes_received" yaml:"bytes_received"`
	BytesSent       int64 `json:"bytes_sent" yaml:"bytes_sent"`
	PacketsReceived int64 `json:"packets_received" yaml:"packets_received"`
	PacketsSent     int64 `json:"packets_sent" yaml:"packets_sent"`
}

// NetworkStateBond represents bond specific state
// API extension: network_state_bond_bridge
type NetworkStateBond struct {
	Mode           string `json:"mode" yaml:"mode"`
	TransmitPolicy string `json:"transmit_policy" yaml:"transmit_policy"`
	UpDelay        uint64 `json:"up_delay" yaml:"up_delay"`
	DownDelay      uint64 `json:"down_delay" yaml:"down_delay"`

	MIIFrequency uint64 `json:"mii_frequency" yaml:"mii_frequency"`
	MIIState     string `json:"mii_state" yaml:"mii_state"`

	LowerDevices []string `json:"lower_devices" yaml:"lower_devices"`
}

// NetworkStateBridge represents bridge specific state
// API extension: network_state_bond_bridge
type NetworkStateBridge struct {
	ID           string `json:"id" yaml:"id"`
	STP          bool   `json:"stp" yaml:"stp"`
	ForwardDelay uint64 `json:"forward_delay" yaml:"forward_delay"`

	VLANDefault   uint64 `json:"vlan_default" yaml:"vlan_default"`
	VLANFiltering bool   `json:"vlan_filtering" yaml:"vlan_filtering"`

	UpperDevices []string `json:"upper_devices" yaml:"upper_devices"`
}

// NetworkStateVLAN represents VLAN specific state
// API extension: network_state_vlan
type NetworkStateVLAN struct {
	LowerDevice string `json:"lower_device" yaml:"lower_device"`
	VID         uint64 `json:"vid" yaml:"vid"`
}
<commit_msg>shared/api: Label Networks structs<commit_after>package api

// NetworksPost represents the fields of a new LXD network
//
// API extension: network
//
// swagger:model
type NetworksPost struct {
	NetworkPut `yaml:",inline"`

	// The name of the new network
	// Example: lxdbr1
	Name string `json:"name" yaml:"name"`

	// The network type (refer to doc/networks.md)
	// Example: bridge
	Type string `json:"type" yaml:"type"`
}

// NetworkPost represents the fields required to rename a LXD network
//
// API extension: network
//
// swagger:model
type NetworkPost struct {
	// The new name for the network
	// Example: lxdbr1
	Name string `json:"name" yaml:"name"`
}

// NetworkPut represents the modifiable fields of a LXD network
//
// API extension: network
//
// swagger:model
type NetworkPut struct {
	// Network configuration map (refer to doc/networks.md)
	// Example: {"ipv4.address": "10.0.0.1/24", "ipv4.nat": "true", "ipv6.address": "none"}
	Config map[string]string `json:"config" yaml:"config"`

	// Description of the profile
	// Example: My new LXD bridge
	//
	// API extension: entity_description
	Description string `json:"description" yaml:"description"`
}

// NetworkStatusPending network is pending creation on other cluster nodes.
const NetworkStatusPending = "Pending"

// NetworkStatusCreated network is fully created.
const NetworkStatusCreated = "Created"

// NetworkStatusErrored network is in error status.
const NetworkStatusErrored = "Errored"

// NetworkStatusUnknown network is in unknown status.
const NetworkStatusUnknown = "Unknown"

// Network represents a LXD network
//
// swagger:model
type Network struct {
	NetworkPut `yaml:",inline"`

	// The network name
	// Read only: true
	// Example: lxdbr0
	Name string `json:"name" yaml:"name"`

	// The network name
	// Read only: true
	// Example: bridge
	Type string `json:"type" yaml:"type"`

	// List of URLs of objects using this profile
	// Read only: true
	// Example: ["/1.0/profiles/default", "/1.0/instances/c1"]
	UsedBy []string `json:"used_by" yaml:"used_by"`

	// Whether this is a LXD managed network
	// Read only: true
	// Example: true
	//
	// API extension: network
	Managed bool `json:"managed" yaml:"managed"`

	// The state of the network (for managed network in clusters)
	// Read only: true
	// Example: Created
	//
	// API extension: clustering
	Status string `json:"status" yaml:"status"`

	// Cluster members on which the network has been defined
	// Read only: true
	// Example: ["lxd01", "lxd02", "lxd03"]
	//
	// API extension: clustering
	Locations []string `json:"locations" yaml:"locations"`
}

// Writable converts a full Network struct into a NetworkPut struct (filters read-only fields)
func (network *Network) Writable() NetworkPut {
	return network.NetworkPut
}

// NetworkLease represents a DHCP lease
//
// API extension: network_leases
//
// swagger:model
type NetworkLease struct {
	// The hostname associated with the record
	// Example: c1
	Hostname string `json:"hostname" yaml:"hostname"`

	// The MAC address
	// Example: 00:16:3e:2c:89:d9
	Hwaddr string `json:"hwaddr" yaml:"hwaddr"`

	// The IP address
	// Example: 10.0.0.98
	Address string `json:"address" yaml:"address"`

	// The type of record (static or dynamic)
	// Example: dynamic
	Type string `json:"type" yaml:"type"`

	// What cluster member this record was found on
	// Example: lxd01
	//
	// API extension: network_leases_location
	Location string `json:"location" yaml:"location"`
}

// NetworkState represents the network state
//
// swagger:model
type NetworkState struct {
	// List of addresses
	Addresses []NetworkStateAddress `json:"addresses" yaml:"addresses"`

	// Interface counters
	Counters NetworkStateCounters `json:"counters" yaml:"counters"`

	// MAC address
	// Example: 00:16:3e:5a:83:57
	Hwaddr string `json:"hwaddr" yaml:"hwaddr"`

	// MTU
	// Example: 1500
	Mtu int `json:"mtu" yaml:"mtu"`

	// Link state
	// Example: up
	State string `json:"state" yaml:"state"`

	// Interface type
	// Example: broadcast
	Type string `json:"type" yaml:"type"`

	// Additional bond interface information
	//
	// API extension: network_state_bond_bridge
	Bond *NetworkStateBond `json:"bond" yaml:"bond"`

	// Additional bridge interface information
	//
	// API extension: network_state_bond_bridge
	Bridge *NetworkStateBridge `json:"bridge" yaml:"bridge"`

	// Additional vlan interface information
	//
	// API extension: network_state_vlan
	VLAN *NetworkStateVLAN `json:"vlan" yaml:"vlan"`
}

// NetworkStateAddress represents a network address
//
// swagger:model
type NetworkStateAddress struct {
	// Address family
	// Example: inet
	Family string `json:"family" yaml:"family"`

	// IP address
	// Example: 10.0.0.1
	Address string `json:"address" yaml:"address"`

	// IP netmask (CIDR)
	// Example: 24
	Netmask string `json:"netmask" yaml:"netmask"`

	// Address scope
	// Example: global
	Scope string `json:"scope" yaml:"scope"`
}

// NetworkStateCounters represents packet counters
//
// swagger:model
type NetworkStateCounters struct {
	// Number of bytes received
	// Example: 250542118
	BytesReceived int64 `json:"bytes_received" yaml:"bytes_received"`

	// Number of bytes sent
	// Example: 17524040140
	BytesSent int64 `json:"bytes_sent" yaml:"bytes_sent"`

	// Number of packets received
	// Example: 1182515
	PacketsReceived int64 `json:"packets_received" yaml:"packets_received"`

	// Number of packets sent
	// Example: 1567934
	PacketsSent int64 `json:"packets_sent" yaml:"packets_sent"`
}

// NetworkStateBond represents bond specific state
//
// API extension: network_state_bond_bridge
//
// swagger:model
type NetworkStateBond struct {
	// Bonding mode
	// Example: 802.3ad
	Mode string `json:"mode" yaml:"mode"`

	// Transmit balancing policy
	// Example: layer3+4
	TransmitPolicy string `json:"transmit_policy" yaml:"transmit_policy"`

	// Delay on link up (ms)
	// Example: 0
	UpDelay uint64 `json:"up_delay" yaml:"up_delay"`

	// Delay on link down (ms)
	// Example: 0
	DownDelay uint64 `json:"down_delay" yaml:"down_delay"`

	// How often to check for link state (ms)
	// Example: 100
	MIIFrequency uint64 `json:"mii_frequency" yaml:"mii_frequency"`

	// Bond link state
	// Example: up
	MIIState string `json:"mii_state" yaml:"mii_state"`

	// List of devices that are part of the bond
	// Example: ["eth0", "eth1"]
	LowerDevices []string `json:"lower_devices" yaml:"lower_devices"`
}

// NetworkStateBridge represents bridge specific state
//
// API extension: network_state_bond_bridge
//
// swagger:model
type NetworkStateBridge struct {
	// Bridge ID
	// Example: 8000.0a0f7c6edbd9
	ID string `json:"id" yaml:"id"`

	// Whether STP is enabled
	// Example: false
	STP bool `json:"stp" yaml:"stp"`

	// Delay on port join (ms)
	// Example: 1500
	ForwardDelay uint64 `json:"forward_delay" yaml:"forward_delay"`

	// Default VLAN ID
	// Example: 1
	VLANDefault uint64 `json:"vlan_default" yaml:"vlan_default"`

	// Whether VLAN filtering is enabled
	// Example: false
	VLANFiltering bool `json:"vlan_filtering" yaml:"vlan_filtering"`

	// List of devices that are in the bridge
	// Example: ["eth0", "eth1"]
	UpperDevices []string `json:"upper_devices" yaml:"upper_devices"`
}

// NetworkStateVLAN represents VLAN specific state
//
// API extension: network_state_vlan
//
// swagger:model
type NetworkStateVLAN struct {
	// Parent device
	// Example: eth0
	LowerDevice string `json:"lower_device" yaml:"lower_device"`

	// VLAN ID
	// Example: 100
	VID uint64 `json:"vid" yaml:"vid"`
}
<|endoftext|>


<commit_before>package shared

import (
	"bufio"
	"fmt"
	"os"
	"os/exec"
	"os/user"
	"path"
	"path/filepath"
	"strconv"
	"strings"
)

/*
 * One entry in id mapping set - a single range of either
 * uid or gid mappings.
 */
type IdmapEntry struct {
	Isuid    bool
	Isgid    bool
	Hostid   int // id as seen on the host - i.e. 100000
	Nsid     int // id as seen in the ns - i.e. 0
	Maprange int
}

func (e *IdmapEntry) ToLxcString() string {
	if e.Isuid {
		return fmt.Sprintf("u %d %d %d", e.Nsid, e.Hostid, e.Maprange)
	}
	return fmt.Sprintf("g %d %d %d", e.Nsid, e.Hostid, e.Maprange)
}

func is_between(x, low, high int) bool {
	return x >= low && x < high
}

func (e *IdmapEntry) Intersects(i IdmapEntry) bool {
	if (e.Isuid && i.Isuid) || (e.Isgid && i.Isgid) {
		switch {
		case is_between(e.Hostid, i.Hostid, i.Hostid+i.Maprange):
			return true
		case is_between(i.Hostid, e.Hostid, e.Hostid+e.Maprange):
			return true
		case is_between(e.Hostid+e.Maprange, i.Hostid, i.Hostid+i.Maprange):
			return true
		case is_between(i.Hostid+e.Maprange, e.Hostid, e.Hostid+e.Maprange):
			return true
		case is_between(e.Nsid, i.Nsid, i.Nsid+i.Maprange):
			return true
		case is_between(i.Nsid, e.Nsid, e.Nsid+e.Maprange):
			return true
		case is_between(e.Nsid+e.Maprange, i.Nsid, i.Nsid+i.Maprange):
			return true
		case is_between(i.Nsid+e.Maprange, e.Nsid, e.Nsid+e.Maprange):
			return true
		}
	}
	return false
}

func (e *IdmapEntry) parse(s string) error {
	split := strings.Split(s, ":")
	var err error
	if len(split) != 4 {
		return fmt.Errorf("Bad idmap: %q", s)
	}
	switch split[0] {
	case "u":
		e.Isuid = true
	case "g":
		e.Isgid = true
	case "b":
		e.Isuid = true
		e.Isgid = true
	default:
		return fmt.Errorf("Bad idmap type in %q", s)
	}
	e.Nsid, err = strconv.Atoi(split[1])
	if err != nil {
		return err
	}
	e.Hostid, err = strconv.Atoi(split[2])
	if err != nil {
		return err
	}
	e.Maprange, err = strconv.Atoi(split[3])
	if err != nil {
		return err
	}

	// wraparound
	if e.Hostid+e.Maprange < e.Hostid || e.Nsid+e.Maprange < e.Nsid {
		return fmt.Errorf("Bad mapping: id wraparound")
	}

	return nil
}

/*
 * Shift a uid from the host into the container
 * I.e. 0 -> 1000 -> 101000
 */
func (e *IdmapEntry) shift_into_ns(id int) (int, error) {
	if id < e.Nsid || id >= e.Nsid+e.Maprange {
		// this mapping doesn't apply
		return 0, fmt.Errorf("N/A")
	}

	return id - e.Nsid + e.Hostid, nil
}

/*
 * Shift a uid from the container back to the host
 * I.e. 101000 -> 1000
 */
func (e *IdmapEntry) shift_from_ns(id int) (int, error) {
	if id < e.Hostid || id >= e.Hostid+e.Maprange {
		// this mapping doesn't apply
		return 0, fmt.Errorf("N/A")
	}

	return id - e.Hostid + e.Nsid, nil
}

/* taken from http://blog.golang.org/slices (which is under BSD licence) */
func Extend(slice []IdmapEntry, element IdmapEntry) []IdmapEntry {
	n := len(slice)
	if n == cap(slice) {
		// Slice is full; must grow.
		// We double its size and add 1, so if the size is zero we still grow.
		newSlice := make([]IdmapEntry, len(slice), 2*len(slice)+1)
		copy(newSlice, slice)
		slice = newSlice
	}
	slice = slice[0 : n+1]
	slice[n] = element
	return slice
}

type IdmapSet struct {
	Idmap []IdmapEntry
}

func (m IdmapSet) Len() int {
	return len(m.Idmap)
}

func (m IdmapSet) Intersects(i IdmapEntry) bool {
	for _, e := range m.Idmap {
		if i.Intersects(e) {
			return true
		}
	}
	return false
}

func (m IdmapSet) ToLxcString() []string {
	var lines []string
	for _, e := range m.Idmap {
		lines = append(lines, e.ToLxcString()+"\n")
	}
	return lines
}

func (m IdmapSet) Append(s string) (IdmapSet, error) {
	e := IdmapEntry{}
	err := e.parse(s)
	if err != nil {
		return m, err
	}
	if m.Intersects(e) {
		return m, fmt.Errorf("Conflicting id mapping")
	}
	m.Idmap = Extend(m.Idmap, e)
	return m, nil
}

func (m IdmapSet) doShiftIntoNs(uid int, gid int, how string) (int, int) {
	u := -1
	g := -1
	for _, e := range m.Idmap {
		var err error
		var tmpu, tmpg int
		if e.Isuid && u == -1 {
			switch how {
			case "in":
				tmpu, err = e.shift_into_ns(uid)
			case "out":
				tmpu, err = e.shift_from_ns(uid)
			}
			if err == nil {
				u = tmpu
			}
		}
		if e.Isgid && g == -1 {
			switch how {
			case "in":
				tmpg, err = e.shift_into_ns(gid)
			case "out":
				tmpg, err = e.shift_from_ns(gid)
			}
			if err == nil {
				g = tmpg
			}
		}
	}

	return u, g
}

func (m IdmapSet) ShiftIntoNs(uid int, gid int) (int, int) {
	return m.doShiftIntoNs(uid, gid, "in")
}

func (m IdmapSet) ShiftFromNs(uid int, gid int) (int, int) {
	return m.doShiftIntoNs(uid, gid, "out")
}

func GetOwner(path string) (int, int, error) {
	uid, gid, _, _, _, _, err := GetFileStat(path)
	return uid, gid, err
}

func (set *IdmapSet) doUidshiftIntoContainer(dir string, testmode bool, how string) error {
	dir = strings.TrimRight(dir, "/")

	convert := func(path string, fi os.FileInfo, err error) (e error) {
		uid, gid, err := GetOwner(path)
		if err != nil {
			return err
		}
		var newuid, newgid int
		switch how {
		case "in":
			newuid, newgid = set.ShiftIntoNs(uid, gid)
		case "out":
			newuid, newgid = set.ShiftFromNs(uid, gid)
		}
		if testmode {
			fmt.Printf("I would shift %q to %d %d\n", path, newuid, newgid)
		} else {
			err = ShiftOwner(dir, path, int(newuid), int(newgid))
			if err != nil {
				return err
			}
		}
		return nil
	}

	if !PathExists(dir) {
		return fmt.Errorf("No such file or directory: %q", dir)
	}
	return filepath.Walk(dir, convert)
}

func (set *IdmapSet) UidshiftIntoContainer(dir string, testmode bool) error {
	return set.doUidshiftIntoContainer(dir, testmode, "in")
}

func (set *IdmapSet) UidshiftFromContainer(dir string, testmode bool) error {
	return set.doUidshiftIntoContainer(dir, testmode, "out")
}

func (set *IdmapSet) ShiftRootfs(p string) error {
	return set.doUidshiftIntoContainer(p, false, "in")
}

func (set *IdmapSet) UnshiftRootfs(p string) error {
	return set.doUidshiftIntoContainer(p, false, "out")
}

const (
	minIDRange = 65536
)

/*
 * get a uid or gid mapping from /etc/subxid
 */
func getFromMap(fname string, username string) (int, int, error) {
	f, err := os.Open(fname)
	var min int
	var idrange int
	if err != nil {
		return 0, 0, err
	}
	defer f.Close()
	scanner := bufio.NewScanner(f)
	min = 0
	idrange = 0
	for scanner.Scan() {
		/*
		 * /etc/sub{gu}id allow comments in the files, so ignore
		 * everything after a '#'
		 */
		s := strings.Split(scanner.Text(), "#")
		if len(s[0]) == 0 {
			continue
		}

		s = strings.Split(s[0], ":")
		if len(s) < 3 {
			return 0, 0, fmt.Errorf("unexpected values in %q: %q", fname, s)
		}
		if strings.EqualFold(s[0], username) {
			bigmin, err := strconv.ParseUint(s[1], 10, 32)
			if err != nil {
				continue
			}
			bigIdrange, err := strconv.ParseUint(s[2], 10, 32)
			if err != nil {
				continue
			}
			min = int(bigmin)
			idrange = int(bigIdrange)
			return min, idrange, nil
		}
	}

	return 0, 0, fmt.Errorf("User %q has no %ss.", username, path.Base(fname))
}

/*
 * Get current username
 */
func getUsername() (string, error) {
	me, err := user.Current()
	if err == nil {
		return me.Username, nil
	} else {
		/* user.Current() requires cgo */
		username := os.Getenv("USER")
		if username == "" {
			return "", err
		}
		return username, nil
	}
}

/*
 * Create a new default idmap
 */
func DefaultIdmapSet() (*IdmapSet, error) {
	myname, err := getUsername()
	if err != nil {
		return nil, err
	}

	umin := 1000000
	urange := 100000
	gmin := 1000000
	grange := 100000

	newuidmap, _ := exec.LookPath("newuidmap")
	newgidmap, _ := exec.LookPath("newgidmap")

	if newuidmap != "" && newgidmap != "" && PathExists("/etc/subuid") && PathExists("/etc/subgid") {
		umin, urange, err = getFromMap("/etc/subuid", myname)
		if err != nil {
			return nil, err
		}

		gmin, grange, err = getFromMap("/etc/subgid", myname)
		if err != nil {
			return nil, err
		}
	}

	if urange < minIDRange {
		return nil, fmt.Errorf("uidrange less than %d", minIDRange)
	}

	if grange < minIDRange {
		return nil, fmt.Errorf("gidrange less than %d", minIDRange)
	}

	m := new(IdmapSet)

	e := IdmapEntry{Isuid: true, Nsid: 0, Hostid: umin, Maprange: urange}
	m.Idmap = Extend(m.Idmap, e)
	e = IdmapEntry{Isgid: true, Nsid: 0, Hostid: gmin, Maprange: grange}
	m.Idmap = Extend(m.Idmap, e)

	return m, nil
}
<commit_msg>Fix idmapset with ZFS<commit_after>package shared

import (
	"bufio"
	"fmt"
	"os"
	"os/exec"
	"os/user"
	"path"
	"path/filepath"
	"strconv"
	"strings"
)

/*
 * One entry in id mapping set - a single range of either
 * uid or gid mappings.
 */
type IdmapEntry struct {
	Isuid    bool
	Isgid    bool
	Hostid   int // id as seen on the host - i.e. 100000
	Nsid     int // id as seen in the ns - i.e. 0
	Maprange int
}

func (e *IdmapEntry) ToLxcString() string {
	if e.Isuid {
		return fmt.Sprintf("u %d %d %d", e.Nsid, e.Hostid, e.Maprange)
	}
	return fmt.Sprintf("g %d %d %d", e.Nsid, e.Hostid, e.Maprange)
}

func is_between(x, low, high int) bool {
	return x >= low && x < high
}

func (e *IdmapEntry) Intersects(i IdmapEntry) bool {
	if (e.Isuid && i.Isuid) || (e.Isgid && i.Isgid) {
		switch {
		case is_between(e.Hostid, i.Hostid, i.Hostid+i.Maprange):
			return true
		case is_between(i.Hostid, e.Hostid, e.Hostid+e.Maprange):
			return true
		case is_between(e.Hostid+e.Maprange, i.Hostid, i.Hostid+i.Maprange):
			return true
		case is_between(i.Hostid+e.Maprange, e.Hostid, e.Hostid+e.Maprange):
			return true
		case is_between(e.Nsid, i.Nsid, i.Nsid+i.Maprange):
			return true
		case is_between(i.Nsid, e.Nsid, e.Nsid+e.Maprange):
			return true
		case is_between(e.Nsid+e.Maprange, i.Nsid, i.Nsid+i.Maprange):
			return true
		case is_between(i.Nsid+e.Maprange, e.Nsid, e.Nsid+e.Maprange):
			return true
		}
	}
	return false
}

func (e *IdmapEntry) parse(s string) error {
	split := strings.Split(s, ":")
	var err error
	if len(split) != 4 {
		return fmt.Errorf("Bad idmap: %q", s)
	}
	switch split[0] {
	case "u":
		e.Isuid = true
	case "g":
		e.Isgid = true
	case "b":
		e.Isuid = true
		e.Isgid = true
	default:
		return fmt.Errorf("Bad idmap type in %q", s)
	}
	e.Nsid, err = strconv.Atoi(split[1])
	if err != nil {
		return err
	}
	e.Hostid, err = strconv.Atoi(split[2])
	if err != nil {
		return err
	}
	e.Maprange, err = strconv.Atoi(split[3])
	if err != nil {
		return err
	}

	// wraparound
	if e.Hostid+e.Maprange < e.Hostid || e.Nsid+e.Maprange < e.Nsid {
		return fmt.Errorf("Bad mapping: id wraparound")
	}

	return nil
}

/*
 * Shift a uid from the host into the container
 * I.e. 0 -> 1000 -> 101000
 */
func (e *IdmapEntry) shift_into_ns(id int) (int, error) {
	if id < e.Nsid || id >= e.Nsid+e.Maprange {
		// this mapping doesn't apply
		return 0, fmt.Errorf("N/A")
	}

	return id - e.Nsid + e.Hostid, nil
}

/*
 * Shift a uid from the container back to the host
 * I.e. 101000 -> 1000
 */
func (e *IdmapEntry) shift_from_ns(id int) (int, error) {
	if id < e.Hostid || id >= e.Hostid+e.Maprange {
		// this mapping doesn't apply
		return 0, fmt.Errorf("N/A")
	}

	return id - e.Hostid + e.Nsid, nil
}

/* taken from http://blog.golang.org/slices (which is under BSD licence) */
func Extend(slice []IdmapEntry, element IdmapEntry) []IdmapEntry {
	n := len(slice)
	if n == cap(slice) {
		// Slice is full; must grow.
		// We double its size and add 1, so if the size is zero we still grow.
		newSlice := make([]IdmapEntry, len(slice), 2*len(slice)+1)
		copy(newSlice, slice)
		slice = newSlice
	}
	slice = slice[0 : n+1]
	slice[n] = element
	return slice
}

type IdmapSet struct {
	Idmap []IdmapEntry
}

func (m IdmapSet) Len() int {
	return len(m.Idmap)
}

func (m IdmapSet) Intersects(i IdmapEntry) bool {
	for _, e := range m.Idmap {
		if i.Intersects(e) {
			return true
		}
	}
	return false
}

func (m IdmapSet) ToLxcString() []string {
	var lines []string
	for _, e := range m.Idmap {
		lines = append(lines, e.ToLxcString()+"\n")
	}
	return lines
}

func (m IdmapSet) Append(s string) (IdmapSet, error) {
	e := IdmapEntry{}
	err := e.parse(s)
	if err != nil {
		return m, err
	}
	if m.Intersects(e) {
		return m, fmt.Errorf("Conflicting id mapping")
	}
	m.Idmap = Extend(m.Idmap, e)
	return m, nil
}

func (m IdmapSet) doShiftIntoNs(uid int, gid int, how string) (int, int) {
	u := -1
	g := -1
	for _, e := range m.Idmap {
		var err error
		var tmpu, tmpg int
		if e.Isuid && u == -1 {
			switch how {
			case "in":
				tmpu, err = e.shift_into_ns(uid)
			case "out":
				tmpu, err = e.shift_from_ns(uid)
			}
			if err == nil {
				u = tmpu
			}
		}
		if e.Isgid && g == -1 {
			switch how {
			case "in":
				tmpg, err = e.shift_into_ns(gid)
			case "out":
				tmpg, err = e.shift_from_ns(gid)
			}
			if err == nil {
				g = tmpg
			}
		}
	}

	return u, g
}

func (m IdmapSet) ShiftIntoNs(uid int, gid int) (int, int) {
	return m.doShiftIntoNs(uid, gid, "in")
}

func (m IdmapSet) ShiftFromNs(uid int, gid int) (int, int) {
	return m.doShiftIntoNs(uid, gid, "out")
}

func GetOwner(path string) (int, int, error) {
	uid, gid, _, _, _, _, err := GetFileStat(path)
	return uid, gid, err
}

func (set *IdmapSet) doUidshiftIntoContainer(dir string, testmode bool, how string) error {
	// Expand any symlink in dir and cleanup resulting path
	dir, err := filepath.EvalSymlinks(dir)
	if err != nil {
		return err
	}
	dir = strings.TrimRight(dir, "/")

	convert := func(path string, fi os.FileInfo, err error) (e error) {
		uid, gid, err := GetOwner(path)
		if err != nil {
			return err
		}
		var newuid, newgid int
		switch how {
		case "in":
			newuid, newgid = set.ShiftIntoNs(uid, gid)
		case "out":
			newuid, newgid = set.ShiftFromNs(uid, gid)
		}
		if testmode {
			fmt.Printf("I would shift %q to %d %d\n", path, newuid, newgid)
		} else {
			err = ShiftOwner(dir, path, int(newuid), int(newgid))
			if err != nil {
				return err
			}
		}
		return nil
	}

	if !PathExists(dir) {
		return fmt.Errorf("No such file or directory: %q", dir)
	}
	return filepath.Walk(dir, convert)
}

func (set *IdmapSet) UidshiftIntoContainer(dir string, testmode bool) error {
	return set.doUidshiftIntoContainer(dir, testmode, "in")
}

func (set *IdmapSet) UidshiftFromContainer(dir string, testmode bool) error {
	return set.doUidshiftIntoContainer(dir, testmode, "out")
}

func (set *IdmapSet) ShiftRootfs(p string) error {
	return set.doUidshiftIntoContainer(p, false, "in")
}

func (set *IdmapSet) UnshiftRootfs(p string) error {
	return set.doUidshiftIntoContainer(p, false, "out")
}

const (
	minIDRange = 65536
)

/*
 * get a uid or gid mapping from /etc/subxid
 */
func getFromMap(fname string, username string) (int, int, error) {
	f, err := os.Open(fname)
	var min int
	var idrange int
	if err != nil {
		return 0, 0, err
	}
	defer f.Close()
	scanner := bufio.NewScanner(f)
	min = 0
	idrange = 0
	for scanner.Scan() {
		/*
		 * /etc/sub{gu}id allow comments in the files, so ignore
		 * everything after a '#'
		 */
		s := strings.Split(scanner.Text(), "#")
		if len(s[0]) == 0 {
			continue
		}

		s = strings.Split(s[0], ":")
		if len(s) < 3 {
			return 0, 0, fmt.Errorf("unexpected values in %q: %q", fname, s)
		}
		if strings.EqualFold(s[0], username) {
			bigmin, err := strconv.ParseUint(s[1], 10, 32)
			if err != nil {
				continue
			}
			bigIdrange, err := strconv.ParseUint(s[2], 10, 32)
			if err != nil {
				continue
			}
			min = int(bigmin)
			idrange = int(bigIdrange)
			return min, idrange, nil
		}
	}

	return 0, 0, fmt.Errorf("User %q has no %ss.", username, path.Base(fname))
}

/*
 * Get current username
 */
func getUsername() (string, error) {
	me, err := user.Current()
	if err == nil {
		return me.Username, nil
	} else {
		/* user.Current() requires cgo */
		username := os.Getenv("USER")
		if username == "" {
			return "", err
		}
		return username, nil
	}
}

/*
 * Create a new default idmap
 */
func DefaultIdmapSet() (*IdmapSet, error) {
	myname, err := getUsername()
	if err != nil {
		return nil, err
	}

	umin := 1000000
	urange := 100000
	gmin := 1000000
	grange := 100000

	newuidmap, _ := exec.LookPath("newuidmap")
	newgidmap, _ := exec.LookPath("newgidmap")

	if newuidmap != "" && newgidmap != "" && PathExists("/etc/subuid") && PathExists("/etc/subgid") {
		umin, urange, err = getFromMap("/etc/subuid", myname)
		if err != nil {
			return nil, err
		}

		gmin, grange, err = getFromMap("/etc/subgid", myname)
		if err != nil {
			return nil, err
		}
	}

	if urange < minIDRange {
		return nil, fmt.Errorf("uidrange less than %d", minIDRange)
	}

	if grange < minIDRange {
		return nil, fmt.Errorf("gidrange less than %d", minIDRange)
	}

	m := new(IdmapSet)

	e := IdmapEntry{Isuid: true, Nsid: 0, Hostid: umin, Maprange: urange}
	m.Idmap = Extend(m.Idmap, e)
	e = IdmapEntry{Isgid: true, Nsid: 0, Hostid: gmin, Maprange: grange}
	m.Idmap = Extend(m.Idmap, e)

	return m, nil
}
<|endoftext|>


<commit_before>package cmd

const Version = "v4.5.6"
<commit_msg>version bump<commit_after>package cmd

const Version = "v4.5.7"
<|endoftext|>