<commit_before>//
// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"testing"

	"github.com/google/go-cmp/cmp"
)

func TestNewCoordinator(t *testing.T) {
	for _, test := range []struct {
		desc    string
		schemas []*tableSchema
		want    []*table
	}{
		{
			desc: "Flat",
			schemas: []*tableSchema{
				{tableName: "A", parentTableName: ""},
				{tableName: "B", parentTableName: ""},
			},
			want: []*table{
				{tableName: "A"},
				{tableName: "B"},
			},
		},
		{
			desc: "Parent-child relationship",
			schemas: []*tableSchema{
				{tableName: "A", parentTableName: ""},
				{tableName: "B", parentTableName: ""},
				{tableName: "C", parentTableName: "B"},
			},
			want: []*table{
				{tableName: "A"},
				{tableName: "B", childTables: []*table{{tableName: "C"}}},
			},
		},
		{
			desc: "Only child table specified",
			schemas: []*tableSchema{
				{tableName: "C", parentTableName: "B"},
			},
			want: []*table{
				{tableName: "C"},
			},
		},
		{
			desc: "Only child table specified in two levels",
			schemas: []*tableSchema{
				{tableName: "C", parentTableName: "B"},
				{tableName: "D", parentTableName: "C"},
			},
			want: []*table{
				{tableName: "C", childTables: []*table{{tableName: "D"}}},
			},
		},
		{
			desc: "Foreign Key reference",
			schemas: []*tableSchema{
				{tableName: "A", parentTableName: ""},
				{tableName: "B", parentTableName: "", referencedBy: []string{}},
				{tableName: "C", parentTableName: "", referencedBy: []string{"B"}},
			},
			want: []*table{
				{tableName: "A"},
				{tableName: "B"},
				{tableName: "C", referencedBy: []*table{{tableName: "B"}}},
			},
		},
	} {
		t.Run(test.desc, func(t *testing.T) {
			coordinator := newCoordinator(test.schemas, nil)
			got := coordinator.tables
			if !compareTables(got, test.want) {
				t.Errorf("invalid tables: got = %#v, want = %#v", got, test.want)
			}
		})
	}
}

func TestFindDeletableTables(t *testing.T) {
	for _, tt := range []struct {
		desc       string
		tablesFunc func() []*table
		want       []string
	}{
		{
			desc: "Flatten tables",
			tablesFunc: func() []*table {
				tableA := &table{tableName: "A", deleter: &deleter{}}
				tableB := &table{tableName: "B", deleter: &deleter{}}
				return []*table{tableA, tableB}
			},
			want: []string{"A", "B"},
		},
		{
			desc: "Parent-child tables with cascade-delete",
			tablesFunc: func() []*table {
				tableA := &table{tableName: "A", deleter: &deleter{}}
				tableB := &table{tableName: "B", deleter: &deleter{}}
				tableC := &table{tableName: "C", deleter: &deleter{}}
				tableB.childTables = []*table{tableC}
				tableC.parentTableName = "B"
				tableC.parentOnDeleteAction = deleteActionCascadeDelete
				return []*table{tableA, tableB}
			},
			want: []string{"A", "B"},
		},
		{
			desc: "Parent-child tables with delete-no-action",
			tablesFunc: func() []*table {
				tableA := &table{tableName: "A", deleter: &deleter{}}
				tableB := &table{tableName: "B", deleter: &deleter{}}
				tableC := &table{tableName: "C", deleter: &deleter{}}
				tableB.childTables = []*table{tableC}
				tableC.parentTableName = "B"
				tableC.parentOnDeleteAction = deleteActionNoAction
				return []*table{tableA, tableB}
			},
			want: []string{"A", "C"},
		},
		{
			desc: "Parent-child tables with delete-no-action, but already child was deleted",
			tablesFunc: func() []*table {
				tableA := &table{tableName: "A", deleter: &deleter{}}
				tableB := &table{tableName: "B", deleter: &deleter{}}
				tableC := &table{tableName: "C", deleter: &deleter{status: statusCompleted}}
				tableB.childTables = []*table{tableC}
				tableC.parentTableName = "B"
				tableC.parentOnDeleteAction = deleteActionNoAction
				return []*table{tableA, tableB}
			},
			want: []string{"A", "B"},
		},
		{
			desc: "Parent-child tables with cascade-delete & delete-no-action",
			tablesFunc: func() []*table {
				tableA := &table{tableName: "A", deleter: &deleter{}}
				tableB := &table{tableName: "B", deleter: &deleter{}}
				tableC := &table{tableName: "C", deleter: &deleter{}}
				tableD := &table{tableName: "D", deleter: &deleter{}}

				// A -- B
				tableA.childTables = []*table{tableB}
				tableB.parentTableName = "A"
				tableB.parentOnDeleteAction = deleteActionCascadeDelete

				// B -- C
				tableB.childTables = []*table{tableC}
				tableC.parentTableName = "B"
				tableC.parentOnDeleteAction = deleteActionNoAction

				// C -- D
				tableC.childTables = []*table{tableD}
				tableD.parentTableName = "C"
				tableD.parentOnDeleteAction = deleteActionCascadeDelete

				return []*table{tableA}
			},
			want: []string{"C"},
		},
		{
			desc: "Foreign key references",
			tablesFunc: func() []*table {
				tableA := &table{tableName: "A", deleter: &deleter{}}
				tableB := &table{tableName: "B", deleter: &deleter{}}
				tableA.referencedBy = []*table{tableB}
				return []*table{tableA, tableB}
			},
			want: []string{"B"},
		},
		{
			desc: "Foreign key references, but referencing table was already deleted",
			tablesFunc: func() []*table {
				tableA := &table{tableName: "A", deleter: &deleter{}}
				tableB := &table{tableName: "B", deleter: &deleter{status: statusCompleted}}
				tableA.referencedBy = []*table{tableB}
				return []*table{tableA, tableB}
			},
			want: []string{"A"},
		},
		{
			desc: "Parent-child tables with foreign key references",
			tablesFunc: func() []*table {
				tableA := &table{tableName: "A", deleter: &deleter{}}
				tableB := &table{tableName: "B", deleter: &deleter{}}
				tableC := &table{tableName: "C", deleter: &deleter{}}
				tableD := &table{tableName: "D", deleter: &deleter{}}

				// A -- B
				tableA.childTables = []*table{tableB}
				tableB.parentTableName = "A"
				tableB.parentOnDeleteAction = deleteActionCascadeDelete

				// C -- D
				tableC.childTables = []*table{tableD}
				tableD.parentTableName = "C"
				tableD.parentOnDeleteAction = deleteActionCascadeDelete

				// Foreign key
				tableB.referencedBy = []*table{tableD}

				return []*table{tableA, tableC}
			},
			want: []string{"C"},
		},
	} {
		t.Run(tt.desc, func(t *testing.T) {
			got := findDeletableTables(tt.tablesFunc())
			gotNames := extractTableNames(got)
			if !cmp.Equal(gotNames, tt.want) {
				t.Errorf("diff(+got, -want) = %v", cmp.Diff(gotNames, tt.want))
			}
		})
	}
}

func TestTableIsDeletable(t *testing.T) {
	for _, tt := range []struct {
		desc      string
		tableFunc func() *table
		want      bool
	}{
		{
			desc: "Simple table",
			tableFunc: func() *table {
				tableA := &table{tableName: "A", deleter: &deleter{}}
				return tableA
			},
			want: true,
		},
		{
			desc: "Child table has cascade-delete action",
			tableFunc: func() *table {
				parent := &table{tableName: "Parent", deleter: &deleter{}}
				child := &table{tableName: "Child", deleter: &deleter{}}
				parent.childTables = []*table{child}
				child.parentTableName = "Parent"
				child.parentOnDeleteAction = deleteActionCascadeDelete
				return parent
			},
			want: true,
		},
		{
			desc: "Child table has no-action action",
			tableFunc: func() *table {
				parent := &table{tableName: "Parent", deleter: &deleter{}}
				child := &table{tableName: "Child", deleter: &deleter{}}
				parent.childTables = []*table{child}
				child.parentTableName = "Parent"
				child.parentOnDeleteAction = deleteActionNoAction
				return parent
			},
			want: false,
		},
		{
			desc: "Child table has no-action action, but it was already deleted",
			tableFunc: func() *table {
				parent := &table{tableName: "Parent", deleter: &deleter{}}
				child := &table{tableName: "Child", deleter: &deleter{status: statusCompleted}}
				parent.childTables = []*table{child}
				child.parentTableName = "Parent"
				child.parentOnDeleteAction = deleteActionNoAction
				return parent
			},
			want: true,
		},
		{
			desc: "Foreign key references",
			tableFunc: func() *table {
				tableA := &table{tableName: "A", deleter: &deleter{}}
				tableB := &table{tableName: "B", deleter: &deleter{}}
				tableA.referencedBy = []*table{tableB}
				return tableA
			},
			want: false,
		},
		{
			desc: "Foreign key references, but referencing table was already deleted",
			tableFunc: func() *table {
				tableA := &table{tableName: "A", deleter: &deleter{}}
				tableB := &table{tableName: "B", deleter: &deleter{status: statusCompleted}}
				tableA.referencedBy = []*table{tableB}
				return tableA
			},
			want: true,
		},
	} {
		t.Run(tt.desc, func(t *testing.T) {
			table := tt.tableFunc()
			if got := table.isDeletable(); got != tt.want {
				t.Errorf("isDeletable(%v) = %v, but want = %v", table, got, tt.want)
			}
		})
	}
}

func extractTableNames(tables []*table) []string {
	names := make([]string, len(tables))
	for i, table := range tables {
		names[i] = table.tableName
	}
	return names
}

func compareTables(tables1, tables2 []*table) bool {
	if len(tables1) != len(tables2) {
		return false
	}
	for i := 0; i < len(tables1); i++ {
		t1 := tables1[i]
		t2 := tables2[i]
		if t1.tableName != t2.tableName {
			return false
		}
		if !compareTables(t1.childTables, t2.childTables) {
			return false
		}
		if !compareTables(t1.referencedBy, t2.referencedBy) {
			return false
		}
	}
	return true
}
<commit_msg>Add more test<commit_after>//
// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"testing"

	"github.com/google/go-cmp/cmp"
)

func TestNewCoordinator(t *testing.T) {
	for _, test := range []struct {
		desc    string
		schemas []*tableSchema
		want    []*table
	}{
		{
			desc: "Flat",
			schemas: []*tableSchema{
				{tableName: "A", parentTableName: ""},
				{tableName: "B", parentTableName: ""},
			},
			want: []*table{
				{tableName: "A"},
				{tableName: "B"},
			},
		},
		{
			desc: "Parent-child relationship",
			schemas: []*tableSchema{
				{tableName: "A", parentTableName: ""},
				{tableName: "B", parentTableName: ""},
				{tableName: "C", parentTableName: "B"},
			},
			want: []*table{
				{tableName: "A"},
				{tableName: "B", childTables: []*table{{tableName: "C"}}},
			},
		},
		{
			desc: "Only child table specified",
			schemas: []*tableSchema{
				{tableName: "C", parentTableName: "B"},
			},
			want: []*table{
				{tableName: "C"},
			},
		},
		{
			desc: "Only child table specified in multiple tables",
			schemas: []*tableSchema{
				{tableName: "C", parentTableName: "B"},
				{tableName: "D", parentTableName: "A"},
			},
			want: []*table{
				{tableName: "C"},
				{tableName: "D"},
			},
		},
		{
			desc: "Only child table specified in two levels",
			schemas: []*tableSchema{
				{tableName: "C", parentTableName: "B"},
				{tableName: "D", parentTableName: "C"},
			},
			want: []*table{
				{tableName: "C", childTables: []*table{{tableName: "D"}}},
			},
		},
		{
			desc: "Foreign Key reference",
			schemas: []*tableSchema{
				{tableName: "A", parentTableName: ""},
				{tableName: "B", parentTableName: "", referencedBy: []string{}},
				{tableName: "C", parentTableName: "", referencedBy: []string{"B"}},
			},
			want: []*table{
				{tableName: "A"},
				{tableName: "B"},
				{tableName: "C", referencedBy: []*table{{tableName: "B"}}},
			},
		},
	} {
		t.Run(test.desc, func(t *testing.T) {
			coordinator := newCoordinator(test.schemas, nil)
			got := coordinator.tables
			if !compareTables(got, test.want) {
				t.Errorf("invalid tables: got = %#v, want = %#v", got, test.want)
			}
		})
	}
}

func TestFindDeletableTables(t *testing.T) {
	for _, tt := range []struct {
		desc       string
		tablesFunc func() []*table
		want       []string
	}{
		{
			desc: "Flatten tables",
			tablesFunc: func() []*table {
				tableA := &table{tableName: "A", deleter: &deleter{}}
				tableB := &table{tableName: "B", deleter: &deleter{}}
				return []*table{tableA, tableB}
			},
			want: []string{"A", "B"},
		},
		{
			desc: "Parent-child tables with cascade-delete",
			tablesFunc: func() []*table {
				tableA := &table{tableName: "A", deleter: &deleter{}}
				tableB := &table{tableName: "B", deleter: &deleter{}}
				tableC := &table{tableName: "C", deleter: &deleter{}}
				tableB.childTables = []*table{tableC}
				tableC.parentTableName = "B"
				tableC.parentOnDeleteAction = deleteActionCascadeDelete
				return []*table{tableA, tableB}
			},
			want: []string{"A", "B"},
		},
		{
			desc: "Parent-child tables with delete-no-action",
			tablesFunc: func() []*table {
				tableA := &table{tableName: "A", deleter: &deleter{}}
				tableB := &table{tableName: "B", deleter: &deleter{}}
				tableC := &table{tableName: "C", deleter: &deleter{}}
				tableB.childTables = []*table{tableC}
				tableC.parentTableName = "B"
				tableC.parentOnDeleteAction = deleteActionNoAction
				return []*table{tableA, tableB}
			},
			want: []string{"A", "C"},
		},
		{
			desc: "Parent-child tables with delete-no-action, but already child was deleted",
			tablesFunc: func() []*table {
				tableA := &table{tableName: "A", deleter: &deleter{}}
				tableB := &table{tableName: "B", deleter: &deleter{}}
				tableC := &table{tableName: "C", deleter: &deleter{status: statusCompleted}}
				tableB.childTables = []*table{tableC}
				tableC.parentTableName = "B"
				tableC.parentOnDeleteAction = deleteActionNoAction
				return []*table{tableA, tableB}
			},
			want: []string{"A", "B"},
		},
		{
			desc: "Parent-child tables with cascade-delete & delete-no-action",
			tablesFunc: func() []*table {
				tableA := &table{tableName: "A", deleter: &deleter{}}
				tableB := &table{tableName: "B", deleter: &deleter{}}
				tableC := &table{tableName: "C", deleter: &deleter{}}
				tableD := &table{tableName: "D", deleter: &deleter{}}

				// A -- B
				tableA.childTables = []*table{tableB}
				tableB.parentTableName = "A"
				tableB.parentOnDeleteAction = deleteActionCascadeDelete

				// B -- C
				tableB.childTables = []*table{tableC}
				tableC.parentTableName = "B"
				tableC.parentOnDeleteAction = deleteActionNoAction

				// C -- D
				tableC.childTables = []*table{tableD}
				tableD.parentTableName = "C"
				tableD.parentOnDeleteAction = deleteActionCascadeDelete

				return []*table{tableA}
			},
			want: []string{"C"},
		},
		{
			desc: "Foreign key references",
			tablesFunc: func() []*table {
				tableA := &table{tableName: "A", deleter: &deleter{}}
				tableB := &table{tableName: "B", deleter: &deleter{}}
				tableA.referencedBy = []*table{tableB}
				return []*table{tableA, tableB}
			},
			want: []string{"B"},
		},
		{
			desc: "Foreign key references, but referencing table was already deleted",
			tablesFunc: func() []*table {
				tableA := &table{tableName: "A", deleter: &deleter{}}
				tableB := &table{tableName: "B", deleter: &deleter{status: statusCompleted}}
				tableA.referencedBy = []*table{tableB}
				return []*table{tableA, tableB}
			},
			want: []string{"A"},
		},
		{
			desc: "Parent-child tables with foreign key references",
			tablesFunc: func() []*table {
				tableA := &table{tableName: "A", deleter: &deleter{}}
				tableB := &table{tableName: "B", deleter: &deleter{}}
				tableC := &table{tableName: "C", deleter: &deleter{}}
				tableD := &table{tableName: "D", deleter: &deleter{}}

				// A -- B
				tableA.childTables = []*table{tableB}
				tableB.parentTableName = "A"
				tableB.parentOnDeleteAction = deleteActionCascadeDelete

				// C -- D
				tableC.childTables = []*table{tableD}
				tableD.parentTableName = "C"
				tableD.parentOnDeleteAction = deleteActionCascadeDelete

				// Foreign key
				tableB.referencedBy = []*table{tableD}

				return []*table{tableA, tableC}
			},
			want: []string{"C"},
		},
	} {
		t.Run(tt.desc, func(t *testing.T) {
			got := findDeletableTables(tt.tablesFunc())
			gotNames := extractTableNames(got)
			if !cmp.Equal(gotNames, tt.want) {
				t.Errorf("diff(+got, -want) = %v", cmp.Diff(gotNames, tt.want))
			}
		})
	}
}

func TestTableIsDeletable(t *testing.T) {
	for _, tt := range []struct {
		desc      string
		tableFunc func() *table
		want      bool
	}{
		{
			desc: "Simple table",
			tableFunc: func() *table {
				tableA := &table{tableName: "A", deleter: &deleter{}}
				return tableA
			},
			want: true,
		},
		{
			desc: "Child table has cascade-delete action",
			tableFunc: func() *table {
				parent := &table{tableName: "Parent", deleter: &deleter{}}
				child := &table{tableName: "Child", deleter: &deleter{}}
				parent.childTables = []*table{child}
				child.parentTableName = "Parent"
				child.parentOnDeleteAction = deleteActionCascadeDelete
				return parent
			},
			want: true,
		},
		{
			desc: "Child table has no-action action",
			tableFunc: func() *table {
				parent := &table{tableName: "Parent", deleter: &deleter{}}
				child := &table{tableName: "Child", deleter: &deleter{}}
				parent.childTables = []*table{child}
				child.parentTableName = "Parent"
				child.parentOnDeleteAction = deleteActionNoAction
				return parent
			},
			want: false,
		},
		{
			desc: "Child table has no-action action, but it was already deleted",
			tableFunc: func() *table {
				parent := &table{tableName: "Parent", deleter: &deleter{}}
				child := &table{tableName: "Child", deleter: &deleter{status: statusCompleted}}
				parent.childTables = []*table{child}
				child.parentTableName = "Parent"
				child.parentOnDeleteAction = deleteActionNoAction
				return parent
			},
			want: true,
		},
		{
			desc: "Foreign key references",
			tableFunc: func() *table {
				tableA := &table{tableName: "A", deleter: &deleter{}}
				tableB := &table{tableName: "B", deleter: &deleter{}}
				tableA.referencedBy = []*table{tableB}
				return tableA
			},
			want: false,
		},
		{
			desc: "Foreign key references, but referencing table was already deleted",
			tableFunc: func() *table {
				tableA := &table{tableName: "A", deleter: &deleter{}}
				tableB := &table{tableName: "B", deleter: &deleter{status: statusCompleted}}
				tableA.referencedBy = []*table{tableB}
				return tableA
			},
			want: true,
		},
	} {
		t.Run(tt.desc, func(t *testing.T) {
			table := tt.tableFunc()
			if got := table.isDeletable(); got != tt.want {
				t.Errorf("isDeletable(%v) = %v, but want = %v", table, got, tt.want)
			}
		})
	}
}

func extractTableNames(tables []*table) []string {
	names := make([]string, len(tables))
	for i, table := range tables {
		names[i] = table.tableName
	}
	return names
}

func compareTables(tables1, tables2 []*table) bool {
	if len(tables1) != len(tables2) {
		return false
	}
	for i := 0; i < len(tables1); i++ {
		t1 := tables1[i]
		t2 := tables2[i]
		if t1.tableName != t2.tableName {
			return false
		}
		if !compareTables(t1.childTables, t2.childTables) {
			return false
		}
		if !compareTables(t1.referencedBy, t2.referencedBy) {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package copy

import (
	"io"

	internalblobinfocache "github.com/containers/image/v5/internal/blobinfocache"
	"github.com/containers/image/v5/pkg/compression"
	compressiontypes "github.com/containers/image/v5/pkg/compression/types"
	"github.com/containers/image/v5/types"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// bpDetectCompressionStepData contains data that the copy pipeline needs about the “detect compression” step.
type bpDetectCompressionStepData struct {
	isCompressed      bool
	format            compressiontypes.Algorithm        // Valid if isCompressed
	decompressor      compressiontypes.DecompressorFunc // Valid if isCompressed
	srcCompressorName string                            // Compressor name to possibly record in the blob info cache for the source blob.
}

// blobPipelineDetectCompressionStep updates *stream to detect its current compression format.
// srcInfo is only used for error messages.
// Returns data for other steps.
func blobPipelineDetectCompressionStep(stream *sourceStream, srcInfo types.BlobInfo) (bpDetectCompressionStepData, error) {
	// This requires us to “peek ahead” into the stream to read the initial part, which requires us to chain through another io.Reader returned by DetectCompression.
	format, decompressor, reader, err := compression.DetectCompressionFormat(stream.reader) // We could skip this in some cases, but let's keep the code path uniform
	if err != nil {
		return bpDetectCompressionStepData{}, errors.Wrapf(err, "reading blob %s", srcInfo.Digest)
	}
	stream.reader = reader

	res := bpDetectCompressionStepData{
		isCompressed: decompressor != nil,
		format:       format,
		decompressor: decompressor,
	}
	if res.isCompressed {
		res.srcCompressorName = format.Name()
	} else {
		res.srcCompressorName = internalblobinfocache.Uncompressed
	}

	if expectedFormat, known := expectedCompressionFormats[stream.info.MediaType]; known && res.isCompressed && format.Name() != expectedFormat.Name() {
		logrus.Debugf("blob %s with type %s should be compressed with %s, but compressor appears to be %s", srcInfo.Digest.String(), srcInfo.MediaType, expectedFormat.Name(), format.Name())
	}
	return res, nil
}

// bpCompressionStepData contains data that the copy pipeline needs about the compression step.
type bpCompressionStepData struct {
	operation              types.LayerCompression      // Operation to use for updating the blob metadata.
	uploadedAlgorithm      *compressiontypes.Algorithm // An algorithm parameter for the compressionOperation edits.
	uploadedAnnotations    map[string]string           // Annotations that should be set on the uploaded blob. WARNING: This is only set after the srcStream.reader is fully consumed.
	srcCompressorName      string                      // Compressor name to record in the blob info cache for the source blob.
	uploadedCompressorName string                      // Compressor name to record in the blob info cache for the uploaded blob.
	closers                []io.Closer                 // Objects to close after the upload is done, if any.
}

// blobPipelineCompressionStep updates *stream to compress and/or decompress it.
// srcInfo is only used for error messages.
// Returns data for other steps; the caller should eventually call updateCompressionEdits and perhaps recordValidatedBlobData,
// and must eventually call close.
func (c *copier) blobPipelineCompressionStep(stream *sourceStream, canModifyBlob bool,
	detected bpDetectCompressionStepData) (*bpCompressionStepData, error) {
	// WARNING: If you are adding new reasons to change the blob, update also the OptimizeDestinationImageAlreadyExists
	// short-circuit conditions
	var closers []io.Closer
	succeeded := false
	defer func() {
		if !succeeded {
			for _, c := range closers {
				c.Close()
			}
		}
	}()
	if canModifyBlob && isOciEncrypted(stream.info.MediaType) {
		// PreserveOriginal due to any compression not being able to be done on an encrypted blob unless decrypted
		logrus.Debugf("Using original blob without modification for encrypted blob")
		succeeded = true
		return &bpCompressionStepData{
			operation:              types.PreserveOriginal,
			uploadedAlgorithm:      nil,
			srcCompressorName:      internalblobinfocache.UnknownCompression,
			uploadedCompressorName: internalblobinfocache.UnknownCompression,
			closers:                closers,
		}, nil
	} else if canModifyBlob && c.dest.DesiredLayerCompression() == types.Compress && !detected.isCompressed {
		logrus.Debugf("Compressing blob on the fly")
		var uploadedAlgorithm *compressiontypes.Algorithm
		if c.compressionFormat != nil {
			uploadedAlgorithm = c.compressionFormat
		} else {
			uploadedAlgorithm = defaultCompressionFormat
		}
		reader, annotations := c.compressedStream(stream.reader, *uploadedAlgorithm)
		closers = append(closers, reader)
		stream.reader = reader
		stream.info = types.BlobInfo{ // FIXME? Should we preserve more data in src.info?
			Digest: "",
			Size:   -1,
		}
		succeeded = true
		return &bpCompressionStepData{
			operation:              types.Compress,
			uploadedAlgorithm:      uploadedAlgorithm,
			uploadedAnnotations:    annotations,
			srcCompressorName:      detected.srcCompressorName,
			uploadedCompressorName: uploadedAlgorithm.Name(),
			closers:                closers,
		}, nil
	}
	if canModifyBlob && c.dest.DesiredLayerCompression() == types.Compress && detected.isCompressed &&
		c.compressionFormat != nil && c.compressionFormat.Name() != detected.format.Name() {
		// When the blob is compressed, but the desired format is different, it first needs to be decompressed and finally
		// re-compressed using the desired format.
		logrus.Debugf("Blob will be converted")

		decompressed, err := detected.decompressor(stream.reader)
		if err != nil {
			return nil, err
		}
		closers = append(closers, decompressed)

		recompressed, annotations := c.compressedStream(decompressed, *c.compressionFormat)
		closers = append(closers, recompressed)
		stream.reader = recompressed
		stream.info = types.BlobInfo{ // FIXME? Should we preserve more data in src.info?
			Digest: "",
			Size:   -1,
		}
		succeeded = true
		return &bpCompressionStepData{
			operation:              types.PreserveOriginal,
			uploadedAlgorithm:      c.compressionFormat,
			uploadedAnnotations:    annotations,
			srcCompressorName:      detected.srcCompressorName,
			uploadedCompressorName: c.compressionFormat.Name(),
			closers:                closers,
		}, nil
	}
	if canModifyBlob && c.dest.DesiredLayerCompression() == types.Decompress && detected.isCompressed {
		logrus.Debugf("Blob will be decompressed")
		s, err := detected.decompressor(stream.reader)
		if err != nil {
			return nil, err
		}
		closers = append(closers, s)
		stream.reader = s
		stream.info = types.BlobInfo{ // FIXME? Should we preserve more data in src.info?
			Digest: "",
			Size:   -1,
		}
		succeeded = true
		return &bpCompressionStepData{
			operation:              types.Decompress,
			uploadedAlgorithm:      nil,
			srcCompressorName:      detected.srcCompressorName,
			uploadedCompressorName: internalblobinfocache.Uncompressed,
			closers:                closers,
		}, nil
	}
	// PreserveOriginal might also need to recompress the original blob if the desired compression format is different.
	logrus.Debugf("Using original blob without modification")
	// Remember if the original blob was compressed, and if so how, so that if
	// LayerInfosForCopy() returned something that differs from what was in the
	// source's manifest, and UpdatedImage() needs to call UpdateLayerInfos(),
	// it will be able to correctly derive the MediaType for the copied blob.
	var algorithm *compressiontypes.Algorithm
	if detected.isCompressed {
		algorithm = &detected.format
	} else {
		algorithm = nil
	}
	succeeded = true
	return &bpCompressionStepData{
		operation:              types.PreserveOriginal,
		uploadedAlgorithm:      algorithm,
		srcCompressorName:      detected.srcCompressorName,
		uploadedCompressorName: detected.srcCompressorName,
		closers:                closers,
	}, nil
}

// updateCompressionEdits sets *operation, *algorithm and updates *annotations, if necessary.
func (d *bpCompressionStepData) updateCompressionEdits(operation *types.LayerCompression, algorithm **compressiontypes.Algorithm, annotations *map[string]string) {
	*operation = d.operation
	// If we can modify the layer's blob, set the desired algorithm for it to be set in the manifest.
	*algorithm = d.uploadedAlgorithm
	if *annotations == nil {
		*annotations = map[string]string{}
	}
	for k, v := range d.uploadedAnnotations {
		(*annotations)[k] = v
	}
}

// recordValidatedBlobData updates b.blobInfoCache with data about the created uploadedInfo adnd the original srcInfo.
// This must ONLY be called if all data has been validated by OUR code, and is not comming from third parties.
func (d *bpCompressionStepData) recordValidatedDigestData(c *copier, uploadedInfo types.BlobInfo, srcInfo types.BlobInfo,
	encryptionStep *bpEncryptionStepData, decryptionStep *bpDecryptionStepData) error {
	// Don’t record any associations that involve encrypted data. This is a bit crude,
	// some blob substitutions (replacing pulls of encrypted data with local reuse of known decryption outcomes)
	// might be safe, but it’s not trivially obvious, so let’s be conservative for now.
	// This crude approach also means we don’t need to record whether a blob is encrypted
	// in the blob info cache (which would probably be necessary for any more complex logic),
	// and the simplicity is attractive.
	if !encryptionStep.encrypting && !decryptionStep.decrypting {
		// If d.operation != types.PreserveOriginal, we now have two reliable digest values:
		// srcinfo.Digest describes the pre-d.operation input, verified by digestingReader
		// uploadedInfo.Digest describes the post-d.operation output, computed by PutBlob
		// (because stream.info.Digest == "", this must have been computed afresh).
		switch d.operation {
		case types.PreserveOriginal:
			break // Do nothing, we have only one digest and we might not have even verified it.
		case types.Compress:
			c.blobInfoCache.RecordDigestUncompressedPair(uploadedInfo.Digest, srcInfo.Digest)
		case types.Decompress:
			c.blobInfoCache.RecordDigestUncompressedPair(srcInfo.Digest, uploadedInfo.Digest)
		default:
			return errors.Errorf("Internal error: Unexpected d.operation value %#v", d.operation)
		}
	}
	if d.uploadedCompressorName != "" && d.uploadedCompressorName != internalblobinfocache.UnknownCompression {
		c.blobInfoCache.RecordDigestCompressorName(uploadedInfo.Digest, d.uploadedCompressorName)
	}
	if srcInfo.Digest != "" && d.srcCompressorName != "" && d.srcCompressorName != internalblobinfocache.UnknownCompression {
		c.blobInfoCache.RecordDigestCompressorName(srcInfo.Digest, d.srcCompressorName)
	}
	return nil
}

// close closes objects that carry state throughout the compression/decompression operation.
func (d *bpCompressionStepData) close() {
	for _, c := range d.closers {
		c.Close()
	}
}

// doCompression reads all input from src and writes its compressed equivalent to dest.
func doCompression(dest io.Writer, src io.Reader, metadata map[string]string, compressionFormat compressiontypes.Algorithm, compressionLevel *int) error {
	compressor, err := compression.CompressStreamWithMetadata(dest, metadata, compressionFormat, compressionLevel)
	if err != nil {
		return err
	}

	buf := make([]byte, compressionBufferSize)

	_, err = io.CopyBuffer(compressor, src, buf) // Sets err to nil, i.e. causes dest.Close()
	if err != nil {
		compressor.Close()
		return err
	}

	return compressor.Close()
}

// compressGoroutine reads all input from src and writes its compressed equivalent to dest.
func (c *copier) compressGoroutine(dest *io.PipeWriter, src io.Reader, metadata map[string]string, compressionFormat compressiontypes.Algorithm) {
	err := errors.New("Internal error: unexpected panic in compressGoroutine")
	defer func() { // Note that this is not the same as {defer dest.CloseWithError(err)}; we need err to be evaluated lazily.
		_ = dest.CloseWithError(err) // CloseWithError(nil) is equivalent to Close(), always returns nil
	}()

	err = doCompression(dest, src, metadata, compressionFormat, c.compressionLevel)
}

// compressedStream returns a stream the input reader compressed using format, and a metadata map.
// The caller must close the returned reader.
// AFTER the stream is consumed, metadata will be updated with annotations to use on the data.
func (c *copier) compressedStream(reader io.Reader, algorithm compressiontypes.Algorithm) (io.ReadCloser, map[string]string) {
	pipeReader, pipeWriter := io.Pipe()
	annotations := map[string]string{}
	// If this fails while writing data, it will do pipeWriter.CloseWithError(); if it fails otherwise,
	// e.g. because we have exited and due to pipeReader.Close() above further writing to the pipe has failed,
	// we don’t care.
	go c.compressGoroutine(pipeWriter, reader, annotations, algorithm) // Closes pipeWriter
	return pipeReader, annotations
}
<commit_msg>Split blobPipelineCompressionStep into individual functions<commit_after>package copy

import (
	"io"

	internalblobinfocache "github.com/containers/image/v5/internal/blobinfocache"
	"github.com/containers/image/v5/pkg/compression"
	compressiontypes "github.com/containers/image/v5/pkg/compression/types"
	"github.com/containers/image/v5/types"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// bpDetectCompressionStepData contains data that the copy pipeline needs about the “detect compression” step.
type bpDetectCompressionStepData struct {
	isCompressed      bool
	format            compressiontypes.Algorithm        // Valid if isCompressed
	decompressor      compressiontypes.DecompressorFunc // Valid if isCompressed
	srcCompressorName string                            // Compressor name to possibly record in the blob info cache for the source blob.
}

// blobPipelineDetectCompressionStep updates *stream to detect its current compression format.
// srcInfo is only used for error messages.
// Returns data for other steps.
func blobPipelineDetectCompressionStep(stream *sourceStream, srcInfo types.BlobInfo) (bpDetectCompressionStepData, error) {
	// This requires us to “peek ahead” into the stream to read the initial part, which requires us to chain through another io.Reader returned by DetectCompression.
	format, decompressor, reader, err := compression.DetectCompressionFormat(stream.reader) // We could skip this in some cases, but let's keep the code path uniform
	if err != nil {
		return bpDetectCompressionStepData{}, errors.Wrapf(err, "reading blob %s", srcInfo.Digest)
	}
	stream.reader = reader

	res := bpDetectCompressionStepData{
		isCompressed: decompressor != nil,
		format:       format,
		decompressor: decompressor,
	}
	if res.isCompressed {
		res.srcCompressorName = format.Name()
	} else {
		res.srcCompressorName = internalblobinfocache.Uncompressed
	}

	if expectedFormat, known := expectedCompressionFormats[stream.info.MediaType]; known && res.isCompressed && format.Name() != expectedFormat.Name() {
		logrus.Debugf("blob %s with type %s should be compressed with %s, but compressor appears to be %s", srcInfo.Digest.String(), srcInfo.MediaType, expectedFormat.Name(), format.Name())
	}
	return res, nil
}

// bpCompressionStepData contains data that the copy pipeline needs about the compression step.
type bpCompressionStepData struct {
	operation              types.LayerCompression      // Operation to use for updating the blob metadata.
	uploadedAlgorithm      *compressiontypes.Algorithm // An algorithm parameter for the compressionOperation edits.
	uploadedAnnotations    map[string]string           // Annotations that should be set on the uploaded blob. WARNING: This is only set after the srcStream.reader is fully consumed.
	srcCompressorName      string                      // Compressor name to record in the blob info cache for the source blob.
	uploadedCompressorName string                      // Compressor name to record in the blob info cache for the uploaded blob.
	closers                []io.Closer                 // Objects to close after the upload is done, if any.
}

// blobPipelineCompressionStep updates *stream to compress and/or decompress it.
// srcInfo is only used for error messages.
// Returns data for other steps; the caller should eventually call updateCompressionEdits and perhaps recordValidatedBlobData,
// and must eventually call close.
func (c *copier) blobPipelineCompressionStep(stream *sourceStream, canModifyBlob bool,
	detected bpDetectCompressionStepData) (*bpCompressionStepData, error) {
	// WARNING: If you are adding new reasons to change the blob, update also the OptimizeDestinationImageAlreadyExists
	// short-circuit conditions
	if canModifyBlob {
		if res := c.bpcPreserveEncrypted(stream); res != nil {
			return res, nil
		}
	}
	if canModifyBlob {
		if res := c.bpcCompressUncompressed(stream, detected); res != nil {
			return res, nil
		}
	}
	if canModifyBlob {
		res, err := c.bpcRecompressCompressed(stream, detected)
		if err != nil {
			return nil, err
		}
		if res != nil {
			return res, nil
		}
	}
	if canModifyBlob {
		res, err := c.bpcDecompressCompressed(stream, detected)
		if err != nil {
			return nil, err
		}
		if res != nil {
			return res, nil
		}
	}
	return c.bpcPreserveOriginal(stream, detected), nil
}

// bpcPreserveEncrypted checks if the input is encrypted, and returns a *bpCompressionStepData if so.
func (c *copier) bpcPreserveEncrypted(stream *sourceStream) *bpCompressionStepData {
	if isOciEncrypted(stream.info.MediaType) {
		// PreserveOriginal due to any compression not being able to be done on an encrypted blob unless decrypted
		logrus.Debugf("Using original blob without modification for encrypted blob")
		return &bpCompressionStepData{
			operation:              types.PreserveOriginal,
			uploadedAlgorithm:      nil,
			srcCompressorName:      internalblobinfocache.UnknownCompression,
			uploadedCompressorName: internalblobinfocache.UnknownCompression,
		}
	}
	return nil
}

// bpcCompressUncompressed checks if we should be compressing an uncompressed input, and returns a *bpCompressionStepData if so.
func (c *copier) bpcCompressUncompressed(stream *sourceStream, detected bpDetectCompressionStepData) *bpCompressionStepData {
	if c.dest.DesiredLayerCompression() == types.Compress && !detected.isCompressed {
		logrus.Debugf("Compressing blob on the fly")
		var uploadedAlgorithm *compressiontypes.Algorithm
		if c.compressionFormat != nil {
			uploadedAlgorithm = c.compressionFormat
		} else {
			uploadedAlgorithm = defaultCompressionFormat
		}

		reader, annotations := c.compressedStream(stream.reader, *uploadedAlgorithm)
		// Note: reader must be closed on all return paths.
		stream.reader = reader
		stream.info = types.BlobInfo{ // FIXME? Should we preserve more data in src.info?
			Digest: "",
			Size:   -1,
		}
		return &bpCompressionStepData{
			operation:              types.Compress,
			uploadedAlgorithm:      uploadedAlgorithm,
			uploadedAnnotations:    annotations,
			srcCompressorName:      detected.srcCompressorName,
			uploadedCompressorName: uploadedAlgorithm.Name(),
			closers:                []io.Closer{reader},
		}
	}
	return nil
}

// bpcRecompressCompressed checks if we should be recompressing a compressed input to another format, and returns a *bpCompressionStepData if so.
func (c *copier) bpcRecompressCompressed(stream *sourceStream, detected bpDetectCompressionStepData) (*bpCompressionStepData, error) {
	if c.dest.DesiredLayerCompression() == types.Compress && detected.isCompressed &&
		c.compressionFormat != nil && c.compressionFormat.Name() != detected.format.Name() {
		// When the blob is compressed, but the desired format is different, it first needs to be decompressed and finally
		// re-compressed using the desired format.
		logrus.Debugf("Blob will be converted")

		var closers []io.Closer
		succeeded := false
		defer func() {
			if !succeeded {
				for _, c := range closers {
					c.Close()
				}
			}
		}()
		decompressed, err := detected.decompressor(stream.reader)
		if err != nil {
			return nil, err
		}
		closers = append(closers, decompressed)

		recompressed, annotations := c.compressedStream(decompressed, *c.compressionFormat)
		closers = append(closers, recompressed)
		stream.reader = recompressed
		stream.info = types.BlobInfo{ // FIXME? Should we preserve more data in src.info?
			Digest: "",
			Size:   -1,
		}
		succeeded = true
		return &bpCompressionStepData{
			operation:              types.PreserveOriginal,
			uploadedAlgorithm:      c.compressionFormat,
			uploadedAnnotations:    annotations,
			srcCompressorName:      detected.srcCompressorName,
			uploadedCompressorName: c.compressionFormat.Name(),
			closers:                closers,
		}, nil
	}
	return nil, nil
}

// bpcDecompressCompressed checks if we should be decompressing a compressed input, and returns a *bpCompressionStepData if so.
func (c *copier) bpcDecompressCompressed(stream *sourceStream, detected bpDetectCompressionStepData) (*bpCompressionStepData, error) {
	if c.dest.DesiredLayerCompression() == types.Decompress && detected.isCompressed {
		logrus.Debugf("Blob will be decompressed")
		s, err := detected.decompressor(stream.reader)
		if err != nil {
			return nil, err
		}
		// Note: s must be closed on all return paths.
		stream.reader = s
		stream.info = types.BlobInfo{ // FIXME? Should we preserve more data in src.info?
			Digest: "",
			Size:   -1,
		}
		return &bpCompressionStepData{
			operation:              types.Decompress,
			uploadedAlgorithm:      nil,
			srcCompressorName:      detected.srcCompressorName,
			uploadedCompressorName: internalblobinfocache.Uncompressed,
			closers:                []io.Closer{s},
		}, nil
	}
	return nil, nil
}

// bpcPreserveOriginal returns a *bpCompressionStepData for not changing the original blob.
func (c *copier) bpcPreserveOriginal(stream *sourceStream, detected bpDetectCompressionStepData) *bpCompressionStepData {
	// PreserveOriginal might also need to recompress the original blob if the desired compression format is different.
	logrus.Debugf("Using original blob without modification")
	// Remember if the original blob was compressed, and if so how, so that if
	// LayerInfosForCopy() returned something that differs from what was in the
	// source's manifest, and UpdatedImage() needs to call UpdateLayerInfos(),
	// it will be able to correctly derive the MediaType for the copied blob.
	var algorithm *compressiontypes.Algorithm
	if detected.isCompressed {
		algorithm = &detected.format
	} else {
		algorithm = nil
	}
	return &bpCompressionStepData{
		operation:              types.PreserveOriginal,
		uploadedAlgorithm:      algorithm,
		srcCompressorName:      detected.srcCompressorName,
		uploadedCompressorName: detected.srcCompressorName,
	}
}

// updateCompressionEdits sets *operation, *algorithm and updates *annotations, if necessary.
func (d *bpCompressionStepData) updateCompressionEdits(operation *types.LayerCompression, algorithm **compressiontypes.Algorithm, annotations *map[string]string) {
	*operation = d.operation
	// If we can modify the layer's blob, set the desired algorithm for it to be set in the manifest.
	*algorithm = d.uploadedAlgorithm
	if *annotations == nil {
		*annotations = map[string]string{}
	}
	for k, v := range d.uploadedAnnotations {
		(*annotations)[k] = v
	}
}

// recordValidatedBlobData updates b.blobInfoCache with data about the created uploadedInfo adnd the original srcInfo.
// This must ONLY be called if all data has been validated by OUR code, and is not comming from third parties.
func (d *bpCompressionStepData) recordValidatedDigestData(c *copier, uploadedInfo types.BlobInfo, srcInfo types.BlobInfo,
	encryptionStep *bpEncryptionStepData, decryptionStep *bpDecryptionStepData) error {
	// Don’t record any associations that involve encrypted data. This is a bit crude,
	// some blob substitutions (replacing pulls of encrypted data with local reuse of known decryption outcomes)
	// might be safe, but it’s not trivially obvious, so let’s be conservative for now.
	// This crude approach also means we don’t need to record whether a blob is encrypted
	// in the blob info cache (which would probably be necessary for any more complex logic),
	// and the simplicity is attractive.
	if !encryptionStep.encrypting && !decryptionStep.decrypting {
		// If d.operation != types.PreserveOriginal, we now have two reliable digest values:
		// srcinfo.Digest describes the pre-d.operation input, verified by digestingReader
		// uploadedInfo.Digest describes the post-d.operation output, computed by PutBlob
		// (because stream.info.Digest == "", this must have been computed afresh).
		switch d.operation {
		case types.PreserveOriginal:
			break // Do nothing, we have only one digest and we might not have even verified it.
		case types.Compress:
			c.blobInfoCache.RecordDigestUncompressedPair(uploadedInfo.Digest, srcInfo.Digest)
		case types.Decompress:
			c.blobInfoCache.RecordDigestUncompressedPair(srcInfo.Digest, uploadedInfo.Digest)
		default:
			return errors.Errorf("Internal error: Unexpected d.operation value %#v", d.operation)
		}
	}
	if d.uploadedCompressorName != "" && d.uploadedCompressorName != internalblobinfocache.UnknownCompression {
		c.blobInfoCache.RecordDigestCompressorName(uploadedInfo.Digest, d.uploadedCompressorName)
	}
	if srcInfo.Digest != "" && d.srcCompressorName != "" && d.srcCompressorName != internalblobinfocache.UnknownCompression {
		c.blobInfoCache.RecordDigestCompressorName(srcInfo.Digest, d.srcCompressorName)
	}
	return nil
}

// close closes objects that carry state throughout the compression/decompression operation.
func (d *bpCompressionStepData) close() {
	for _, c := range d.closers {
		c.Close()
	}
}

// doCompression reads all input from src and writes its compressed equivalent to dest.
func doCompression(dest io.Writer, src io.Reader, metadata map[string]string, compressionFormat compressiontypes.Algorithm, compressionLevel *int) error {
	compressor, err := compression.CompressStreamWithMetadata(dest, metadata, compressionFormat, compressionLevel)
	if err != nil {
		return err
	}

	buf := make([]byte, compressionBufferSize)

	_, err = io.CopyBuffer(compressor, src, buf) // Sets err to nil, i.e. causes dest.Close()
	if err != nil {
		compressor.Close()
		return err
	}

	return compressor.Close()
}

// compressGoroutine reads all input from src and writes its compressed equivalent to dest.
func (c *copier) compressGoroutine(dest *io.PipeWriter, src io.Reader, metadata map[string]string, compressionFormat compressiontypes.Algorithm) {
	err := errors.New("Internal error: unexpected panic in compressGoroutine")
	defer func() { // Note that this is not the same as {defer dest.CloseWithError(err)}; we need err to be evaluated lazily.
		_ = dest.CloseWithError(err) // CloseWithError(nil) is equivalent to Close(), always returns nil
	}()

	err = doCompression(dest, src, metadata, compressionFormat, c.compressionLevel)
}

// compressedStream returns a stream the input reader compressed using format, and a metadata map.
// The caller must close the returned reader.
// AFTER the stream is consumed, metadata will be updated with annotations to use on the data.
func (c *copier) compressedStream(reader io.Reader, algorithm compressiontypes.Algorithm) (io.ReadCloser, map[string]string) {
	pipeReader, pipeWriter := io.Pipe()
	annotations := map[string]string{}
	// If this fails while writing data, it will do pipeWriter.CloseWithError(); if it fails otherwise,
	// e.g. because we have exited and due to pipeReader.Close() above further writing to the pipe has failed,
	// we don’t care.
	go c.compressGoroutine(pipeWriter, reader, annotations, algorithm) // Closes pipeWriter
	return pipeReader, annotations
}
<|endoftext|>


<commit_before>package concordances

import (
	"fmt"

	"github.com/Financial-Times/neo-model-utils-go/mapper"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	log "github.com/Sirupsen/logrus"
	"github.com/jmcvetta/neoism"
)

// Driver interface
type Driver interface {
	ReadByConceptID(ids []string) (concordances Concordances, found bool, err error)
	ReadByAuthority(authority string, ids []string) (concordances Concordances, found bool, err error)
	CheckConnectivity() error
}

// CypherDriver struct
type CypherDriver struct {
	conn neoutils.NeoConnection
	env  string
}

//NewCypherDriver instantiate driver
func NewCypherDriver(conn neoutils.NeoConnection, env string) CypherDriver {
	return CypherDriver{conn, env}
}

// CheckConnectivity tests neo4j by running a simple cypher query
func (pcw CypherDriver) CheckConnectivity() error {
	return neoutils.Check(pcw.conn)
}

type neoReadStruct struct {
	UUID                 string        `json:"UUID"`
	Types                []string      `json:"TYPES"`
	NeoIdentifier        neoIdentifier `json:"IDENTIFIERS"`
	PrefUUID             string        `json:"prefUUID"`
	canonicalTypes       []string      `json:"canonicalTypes"`
	canonicalIdentifiers neoIdentifier `json:"canonicalIdentifiers"`
}

type neoIdentifier struct {
	Labels []string `json:"labels"`
	Value  string   `json:"value"`
}

type neoResultStrunct struct {
	Rs []neoReadStruct
}

func (pcw CypherDriver) ReadByConceptID(identifiers []string) (concordances Concordances, found bool, err error) {
	results := []neoReadStruct{}
	query := &neoism.CypherQuery{
		Statement: `
		MATCH (p:Concept)<-[:IDENTIFIES]-(i:UPPIdentifier)
		WHERE i.value in {identifiers}
		MATCH (p:Concept)<-[:IDENTIFIES]-(ids:Identifier)
		WHERE NOT ids:UPPIdentifier
        OPTIONAL MATCH (p:Concept)-[:EQUIVALENT_TO]->(canonical:Concept)
        OPTIONAL MATCH (leafNode:Concept)-[:EQUIVALENT_TO]->(canonical:Concept)
        OPTIONAL MATCH (leafNode:Concept)<-[:IDENTIFIES]-(leafId:Identifier)
        WHERE NOT leafId:UPPIdentifier
	    RETURN canonical.prefUUID as prefUUID, p.uuid AS UUID, labels(p) AS TYPES, labels(p) AS canonicalTypes, {labels:labels(ids), value:ids.value} as IDENTIFIERS,
        {labels:labels(leafId), value:leafId.value} as canonicalIdentifiers`,
		Parameters: neoism.Props{"identifiers": identifiers},
		Result:     &results,
	}

	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		log.Errorf("Error looking up Concordances with query %s from neoism: %+v\n", query.Statement, err)
		return Concordances{}, false, fmt.Errorf("Error accessing Concordance datastore for identifier:")
	}

	if (len(results)) == 0 {
		return Concordances{}, false, nil
	}

	concordances = Concordances{
		Concordance: []Concordance{},
	}

	return processCypherQueryToConcordances(pcw, query, &results)

}

func (pcw CypherDriver) ReadByAuthority(authority string, identifierValues []string) (concordances Concordances, found bool, err error) {
	results := []neoReadStruct{}

	authorityProperty := mapAuthorityToAuthorityProperty(authority)
	if authorityProperty == "" {
		return Concordances{}, false, nil
	}
	query := &neoism.CypherQuery{
		Statement: `
		MATCH (p:Concept)<-[:IDENTIFIES]-(ids:Identifier)
 		WHERE ids.value in {identifierValues} AND NOT ids:UPPIdentifier
        OPTIONAL MATCH (p:Concept)-[:EQUIVALENT_TO]->(canonical:Concept)
        RETURN canonical.prefUUID as prefUUID, p.uuid AS UUID, labels(p) AS TYPES, labels(canonical) AS canonicalTypes, {labels:labels(ids), value:ids.value} as IDENTIFIERS,
        {labels:collect(p.authority), value:p.authorityValue} as canonicalIdentifiers`,
		Parameters: neoism.Props{
			"identifierValues": identifierValues,
			"authority":        authorityProperty,
		},
		Result: &results,
	}

	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		log.Errorf("Error looking up Concordances with query %s from neoism: %+v\n", query.Statement, err)
		return Concordances{}, false, fmt.Errorf("Error accessing Concordance datastore for identifier:")
	}

	if (len(results)) == 0 {
		return Concordances{}, false, nil
	}

	concordances = Concordances{
		Concordance: []Concordance{},
	}

	return processCypherQueryToConcordances(pcw, query, &results)
}

func processCypherQueryToConcordances(pcw CypherDriver, q *neoism.CypherQuery, results *[]neoReadStruct) (concordances Concordances, found bool, err error) {
	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{q})
	if err != nil {
		log.Errorf("Error looking up Concordances with query %s from neoism: %+v\n", q.Statement, err)
		return Concordances{}, false, fmt.Errorf("Error accessing Concordance datastore")
	}

	concordances = neoReadStructToConcordances(results, pcw.env)

	if (len(concordances.Concordance)) == 0 {
		return Concordances{}, false, nil
	}
	return concordances, true, nil
}

func neoReadStructToConcordances(neo *[]neoReadStruct, env string) (concordances Concordances) {
	concordances = Concordances{
		Concordance: []Concordance{},
	}
	for _, neoCon := range *neo {
		var con = Concordance{}
		var concept = Concept{}

		if neoCon.PrefUUID != "" {
			log.Debugf("New concept model with prefUUID: %v", neoCon.PrefUUID)
			concept.ID = neoCon.PrefUUID
			con.Identifier = Identifier{Authority: mapNeoLabelsToAuthorityValue(neoCon.canonicalIdentifiers.Labels), IdentifierValue: neoCon.canonicalIdentifiers.Value}
			concept.APIURL = mapper.APIURL(neoCon.PrefUUID, neoCon.canonicalTypes, env)
		} else {
			log.Debugf("Old concept model with UUID: %v", neoCon.UUID)
			concept.ID = mapper.IDURL(neoCon.UUID)
			con.Identifier = Identifier{Authority: mapNeoLabelsToAuthorityValue(neoCon.NeoIdentifier.Labels), IdentifierValue: neoCon.NeoIdentifier.Value}
			concept.APIURL = mapper.APIURL(neoCon.UUID, neoCon.Types, env)
		}

		con.Concept = concept
		concordances.Concordance = append(concordances.Concordance, con)
	}
	return concordances
}

func mapNeoLabelsToAuthorityValue(labelNames []string) (authority string) {
	for _, label := range labelNames {
		switch label {
		// Old style node label lookup
		case TME_ID_NODE_LABEL:
			return TME_AUTHORITY
		case FS_ID_NODE_LABEL:
			return FS_AUTHORITY
		case UP_ID_NODE_LABEL:
			return UP_AUTHORITY
		case LEI_ID_NODE_LABEL:
			return LEI_AUTHORITY
		case SL_ID_NODE_LABEL:
			return SL_AUTHORITY

		// New style authority properties
		case FS_AUTHORITY_PROPERTY:
			return FS_AUTHORITY
		case UP_AUTHORITY_PROPERTY:
			return UP_AUTHORITY
		case SL_AUTHORITY_PROPERTY:
			return SL_AUTHORITY
		case TME_AUTHORITY_PROPERTY:
			return TME_AUTHORITY
		}
	}
	return ""
}

func mapAuthorityToAuthorityProperty(authority string) string {
	switch authority {
	case TME_AUTHORITY:
		return TME_AUTHORITY_PROPERTY
	case FS_AUTHORITY:
		return FS_AUTHORITY_PROPERTY
	case SL_AUTHORITY:
		return SL_AUTHORITY_PROPERTY
	default:
		return ""
	}
}

func mapAuthorityToIdentifierLabel(authority string) (label string) {
	switch authority {
	case TME_AUTHORITY:
		return TME_ID_NODE_LABEL
	case FS_AUTHORITY:
		return FS_ID_NODE_LABEL
	case LEI_AUTHORITY:
		return LEI_ID_NODE_LABEL
	}
	return ""
}

const TME_AUTHORITY = "http://api.ft.com/system/FT-TME"
const FS_AUTHORITY = "http://api.ft.com/system/FACTSET"
const UP_AUTHORITY = "http://api.ft.com/system/UPP"
const LEI_AUTHORITY = "http://api.ft.com/system/LEI"
const SL_AUTHORITY = "http://api.ft.com/system/SMARTLOGIC"

const TME_ID_NODE_LABEL = "TMEIdentifier"
const FS_ID_NODE_LABEL = "FactsetIdentifier"
const UP_ID_NODE_LABEL = "UPPIdentifier"
const LEI_ID_NODE_LABEL = "LegalEntityIdentifier"
const SL_ID_NODE_LABEL = "SmartlogicIdentifier"

const TME_AUTHORITY_PROPERTY = "TME"
const FS_AUTHORITY_PROPERTY = "FACTSET"
const UP_AUTHORITY_PROPERTY = "UPP"
const SL_AUTHORITY_PROPERTY = "Smartlogic"
<commit_msg>Fixed authority issue<commit_after>package concordances

import (
	"fmt"

	"github.com/Financial-Times/neo-model-utils-go/mapper"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	log "github.com/Sirupsen/logrus"
	"github.com/jmcvetta/neoism"
)

// Driver interface
type Driver interface {
	ReadByConceptID(ids []string) (concordances Concordances, found bool, err error)
	ReadByAuthority(authority string, ids []string) (concordances Concordances, found bool, err error)
	CheckConnectivity() error
}

// CypherDriver struct
type CypherDriver struct {
	conn neoutils.NeoConnection
	env  string
}

//NewCypherDriver instantiate driver
func NewCypherDriver(conn neoutils.NeoConnection, env string) CypherDriver {
	return CypherDriver{conn, env}
}

// CheckConnectivity tests neo4j by running a simple cypher query
func (pcw CypherDriver) CheckConnectivity() error {
	return neoutils.Check(pcw.conn)
}

type neoReadStruct struct {
	UUID                 string        `json:"UUID"`
	Types                []string      `json:"TYPES"`
	NeoIdentifier        neoIdentifier `json:"IDENTIFIERS"`
	PrefUUID             string        `json:"prefUUID"`
	canonicalTypes       []string      `json:"canonicalTypes"`
	canonicalIdentifiers neoIdentifier `json:"canonicalIdentifiers"`
}

type neoIdentifier struct {
	Labels []string `json:"labels"`
	Value  string   `json:"value"`
}

type neoResultStrunct struct {
	Rs []neoReadStruct
}

func (pcw CypherDriver) ReadByConceptID(identifiers []string) (concordances Concordances, found bool, err error) {
	results := []neoReadStruct{}
	query := &neoism.CypherQuery{
		Statement: `
		MATCH (p:Concept)<-[:IDENTIFIES]-(i:UPPIdentifier)
		WHERE i.value in {identifiers}
		MATCH (p:Concept)<-[:IDENTIFIES]-(ids:Identifier)
		WHERE NOT ids:UPPIdentifier
        OPTIONAL MATCH (p:Concept)-[:EQUIVALENT_TO]->(canonical:Concept)
        OPTIONAL MATCH (leafNode:Concept)-[:EQUIVALENT_TO]->(canonical:Concept)
        OPTIONAL MATCH (leafNode:Concept)<-[:IDENTIFIES]-(leafId:Identifier)
        WHERE NOT leafId:UPPIdentifier
	    RETURN canonical.prefUUID as prefUUID, p.uuid AS UUID, labels(p) AS TYPES, labels(p) AS canonicalTypes, {labels:labels(ids), value:ids.value} as IDENTIFIERS,
        {labels:labels(leafId), value:leafId.value} as canonicalIdentifiers`,
		Parameters: neoism.Props{"identifiers": identifiers},
		Result:     &results,
	}

	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		log.Errorf("Error looking up Concordances with query %s from neoism: %+v\n", query.Statement, err)
		return Concordances{}, false, fmt.Errorf("Error accessing Concordance datastore for identifier:")
	}

	if (len(results)) == 0 {
		return Concordances{}, false, nil
	}

	concordances = Concordances{
		Concordance: []Concordance{},
	}

	return processCypherQueryToConcordances(pcw, query, &results)

}

func (pcw CypherDriver) ReadByAuthority(authority string, identifierValues []string) (concordances Concordances, found bool, err error) {
	results := []neoReadStruct{}

	authorityProperty := mapAuthorityToAuthorityProperty(authority)
	if authorityProperty == "" {
		return Concordances{}, false, nil
	}
	query := &neoism.CypherQuery{
		Statement: `
		MATCH (p:Concept)<-[:IDENTIFIES]-(ids:Identifier)
 		WHERE ids.value in {identifierValues} AND NOT ids:UPPIdentifier
        OPTIONAL MATCH (p:Concept)-[:EQUIVALENT_TO]->(canonical:Concept)
        RETURN canonical.prefUUID as prefUUID, p.uuid AS UUID, labels(p) AS TYPES, labels(canonical) AS canonicalTypes, {labels:labels(ids), value:ids.value} as IDENTIFIERS`,
		Parameters: neoism.Props{
			"identifierValues": identifierValues,
			"authority":        authorityProperty,
		},
		Result: &results,
	}

	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		log.Errorf("Error looking up Concordances with query %s from neoism: %+v\n", query.Statement, err)
		return Concordances{}, false, fmt.Errorf("Error accessing Concordance datastore for identifier:")
	}

	if (len(results)) == 0 {
		return Concordances{}, false, nil
	}

	concordances = Concordances{
		Concordance: []Concordance{},
	}

	return processCypherQueryToConcordances(pcw, query, &results)
}

func processCypherQueryToConcordances(pcw CypherDriver, q *neoism.CypherQuery, results *[]neoReadStruct) (concordances Concordances, found bool, err error) {
	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{q})
	if err != nil {
		log.Errorf("Error looking up Concordances with query %s from neoism: %+v\n", q.Statement, err)
		return Concordances{}, false, fmt.Errorf("Error accessing Concordance datastore")
	}

	concordances = neoReadStructToConcordances(results, pcw.env)

	if (len(concordances.Concordance)) == 0 {
		return Concordances{}, false, nil
	}
	return concordances, true, nil
}

func neoReadStructToConcordances(neo *[]neoReadStruct, env string) (concordances Concordances) {
	concordances = Concordances{
		Concordance: []Concordance{},
	}
	for _, neoCon := range *neo {
		var con = Concordance{}
		var concept = Concept{}

		if neoCon.PrefUUID != "" {
			log.Debugf("New concept model with prefUUID: %v", neoCon.PrefUUID)
			concept.ID = neoCon.PrefUUID
			concept.APIURL = mapper.APIURL(neoCon.PrefUUID, neoCon.canonicalTypes, env)
		} else {
			log.Debugf("Old concept model with UUID: %v", neoCon.UUID)
			concept.ID = mapper.IDURL(neoCon.UUID)
			concept.APIURL = mapper.APIURL(neoCon.UUID, neoCon.Types, env)
		}
		con.Identifier = Identifier{Authority: mapNeoLabelsToAuthorityValue(neoCon.NeoIdentifier.Labels), IdentifierValue: neoCon.NeoIdentifier.Value}
		con.Concept = concept
		concordances.Concordance = append(concordances.Concordance, con)
	}
	return concordances
}

func mapNeoLabelsToAuthorityValue(labelNames []string) (authority string) {
	for _, label := range labelNames {
		switch label {
		// Old style node label lookup
		case TME_ID_NODE_LABEL:
			return TME_AUTHORITY
		case FS_ID_NODE_LABEL:
			return FS_AUTHORITY
		case UP_ID_NODE_LABEL:
			return UP_AUTHORITY
		case LEI_ID_NODE_LABEL:
			return LEI_AUTHORITY
		case SL_ID_NODE_LABEL:
			return SL_AUTHORITY

		// New style authority properties
		case FS_AUTHORITY_PROPERTY:
			return FS_AUTHORITY
		case UP_AUTHORITY_PROPERTY:
			return UP_AUTHORITY
		case SL_AUTHORITY_PROPERTY:
			return SL_AUTHORITY
		case TME_AUTHORITY_PROPERTY:
			return TME_AUTHORITY
		}
	}
	return ""
}

func mapAuthorityToAuthorityProperty(authority string) string {
	switch authority {
	case TME_AUTHORITY:
		return TME_AUTHORITY_PROPERTY
	case FS_AUTHORITY:
		return FS_AUTHORITY_PROPERTY
	case SL_AUTHORITY:
		return SL_AUTHORITY_PROPERTY
	default:
		return ""
	}
}

func mapAuthorityToIdentifierLabel(authority string) (label string) {
	switch authority {
	case TME_AUTHORITY:
		return TME_ID_NODE_LABEL
	case FS_AUTHORITY:
		return FS_ID_NODE_LABEL
	case LEI_AUTHORITY:
		return LEI_ID_NODE_LABEL
	}
	return ""
}

const TME_AUTHORITY = "http://api.ft.com/system/FT-TME"
const FS_AUTHORITY = "http://api.ft.com/system/FACTSET"
const UP_AUTHORITY = "http://api.ft.com/system/UPP"
const LEI_AUTHORITY = "http://api.ft.com/system/LEI"
const SL_AUTHORITY = "http://api.ft.com/system/SMARTLOGIC"

const TME_ID_NODE_LABEL = "TMEIdentifier"
const FS_ID_NODE_LABEL = "FactsetIdentifier"
const UP_ID_NODE_LABEL = "UPPIdentifier"
const LEI_ID_NODE_LABEL = "LegalEntityIdentifier"
const SL_ID_NODE_LABEL = "SmartlogicIdentifier"

const TME_AUTHORITY_PROPERTY = "TME"
const FS_AUTHORITY_PROPERTY = "FACTSET"
const UP_AUTHORITY_PROPERTY = "UPP"
const SL_AUTHORITY_PROPERTY = "Smartlogic"
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package bcrypt implements Provos and Mazières's bcrypt adapative hashing
// algorithm. See http://www.usenix.org/event/usenix99/provos/provos.pdf
package bcrypt

// The code is a port of Provos and Mazières's C implementation.
import (
	"crypto/blowfish"
	"crypto/rand"
	"crypto/subtle"
	"errors"
	"fmt"
	"io"
	"strconv"
)

const (
	MinCost     int = 4  // the minimum allowable cost as passed in to GenerateFromPassword
	MaxCost     int = 31 // the maximum allowable cost as passed in to GenerateFromPassword
	DefaultCost int = 10 // the cost that will actually be set if a cost below MinCost is passed into GenerateFromPassword
)

// The error returned from CompareHashAndPassword when a password and hash do
// not match.
var MismatchedHashAndPasswordError = errors.New("crypto/bcrypt: hashedPassword is not the hash of the given password")

// The error returned from CompareHashAndPassword when a hash is too short to
// be a bcrypt hash.
var HashTooShortError = errors.New("crypto/bcrypt: hashedSecret too short to be a bcrypted password")

// The error returned from CompareHashAndPassword when a hash was created with
// a bcrypt algorithm newer than this implementation.
type HashVersionTooNewError byte

func (hv HashVersionTooNewError) Error() string {
	return fmt.Sprintf("crypto/bcrypt: bcrypt algorithm version '%c' requested is newer than current version '%c'", byte(hv), majorVersion)
}

// The error returned from CompareHashAndPassword when a hash starts with something other than '$'
type InvalidHashPrefixError byte

func (ih InvalidHashPrefixError) Error() string {
	return fmt.Sprintf("crypto/bcrypt: bcrypt hashes must start with '$', but hashedSecret started with '%c'", byte(ih))
}

type InvalidCostError int

func (ic InvalidCostError) Error() string {
	return fmt.Sprintf("crypto/bcrypt: cost %d is outside allowed range (%d,%d)", int(ic), int(MinCost), int(MaxCost))
}

const (
	majorVersion       = '2'
	minorVersion       = 'a'
	maxSaltSize        = 16
	maxCryptedHashSize = 23
	encodedSaltSize    = 22
	encodedHashSize    = 31
	minHashSize        = 59
)

// magicCipherData is an IV for the 64 Blowfish encryption calls in
// bcrypt(). It's the string "OrpheanBeholderScryDoubt" in big-endian bytes.
var magicCipherData = []byte{
	0x4f, 0x72, 0x70, 0x68,
	0x65, 0x61, 0x6e, 0x42,
	0x65, 0x68, 0x6f, 0x6c,
	0x64, 0x65, 0x72, 0x53,
	0x63, 0x72, 0x79, 0x44,
	0x6f, 0x75, 0x62, 0x74,
}

type hashed struct {
	hash  []byte
	salt  []byte
	cost  uint32 // allowed range is MinCost to MaxCost
	major byte
	minor byte
}

// GenerateFromPassword returns the bcrypt hash of the password at the given
// cost. If the cost given is less than MinCost, the cost will be set to
// MinCost, instead. Use CompareHashAndPassword, as defined in this package,
// to compare the returned hashed password with its cleartext version.
func GenerateFromPassword(password []byte, cost int) ([]byte, error) {
	p, err := newFromPassword(password, cost)
	if err != nil {
		return nil, err
	}
	return p.Hash(), nil
}

// CompareHashAndPassword compares a bcrypt hashed password with its possible
// plaintext equivalent. Note: Using bytes.Equal for this job is
// insecure. Returns nil on success, or an error on failure.
func CompareHashAndPassword(hashedPassword, password []byte) error {
	p, err := newFromHash(hashedPassword)
	if err != nil {
		return err
	}

	otherHash, err := bcrypt(password, p.cost, p.salt)
	if err != nil {
		return err
	}

	otherP := &hashed{otherHash, p.salt, p.cost, p.major, p.minor}
	if subtle.ConstantTimeCompare(p.Hash(), otherP.Hash()) == 1 {
		return nil
	}

	return MismatchedHashAndPasswordError
}

func newFromPassword(password []byte, cost int) (*hashed, error) {
	if cost < MinCost {
		cost = DefaultCost
	}
	p := new(hashed)
	p.major = majorVersion
	p.minor = minorVersion

	err := checkCost(cost)
	if err != nil {
		return nil, err
	}
	p.cost = uint32(cost)

	unencodedSalt := make([]byte, maxSaltSize)
	_, err = io.ReadFull(rand.Reader, unencodedSalt)
	if err != nil {
		return nil, err
	}

	p.salt = base64Encode(unencodedSalt)
	hash, err := bcrypt(password, p.cost, p.salt)
	if err != nil {
		return nil, err
	}
	p.hash = hash
	return p, err
}

func newFromHash(hashedSecret []byte) (*hashed, error) {
	if len(hashedSecret) < minHashSize {
		return nil, HashTooShortError
	}
	p := new(hashed)
	n, err := p.decodeVersion(hashedSecret)
	if err != nil {
		return nil, err
	}
	hashedSecret = hashedSecret[n:]
	n, err = p.decodeCost(hashedSecret)
	if err != nil {
		return nil, err
	}
	hashedSecret = hashedSecret[n:]

	// The "+2" is here because we'll have to append at most 2 '=' to the salt
	// when base64 decoding it in expensiveBlowfishSetup().
	p.salt = make([]byte, encodedSaltSize, encodedSaltSize+2)
	copy(p.salt, hashedSecret[:encodedSaltSize])

	hashedSecret = hashedSecret[encodedSaltSize:]
	p.hash = make([]byte, len(hashedSecret))
	copy(p.hash, hashedSecret)

	return p, nil
}

func bcrypt(password []byte, cost uint32, salt []byte) ([]byte, error) {
	cipherData := make([]byte, len(magicCipherData))
	copy(cipherData, magicCipherData)

	c, err := expensiveBlowfishSetup(password, cost, salt)
	if err != nil {
		return nil, err
	}

	for i := 0; i < 24; i += 8 {
		for j := 0; j < 64; j++ {
			c.Encrypt(cipherData[i:i+8], cipherData[i:i+8])
		}
	}

	// Bug compatibility with C bcrypt implementations. We only encode 23 of
	// the 24 bytes encrypted.
	hsh := base64Encode(cipherData[:maxCryptedHashSize])
	return hsh, nil
}

func expensiveBlowfishSetup(key []byte, cost uint32, salt []byte) (*blowfish.Cipher, error) {

	csalt, err := base64Decode(salt)
	if err != nil {
		return nil, err
	}

	// Bug compatibility with C bcrypt implementations. They use the trailing
	// NULL in the key string during expansion.
	ckey := append(key, 0)

	c, err := blowfish.NewSaltedCipher(ckey, csalt)
	if err != nil {
		return nil, err
	}

	rounds := 1 << cost
	for i := 0; i < rounds; i++ {
		blowfish.ExpandKey(ckey, c)
		blowfish.ExpandKey(csalt, c)
	}

	return c, nil
}

func (p *hashed) Hash() []byte {
	arr := make([]byte, 60)
	arr[0] = '$'
	arr[1] = p.major
	n := 2
	if p.minor != 0 {
		arr[2] = p.minor
		n = 3
	}
	arr[n] = '$'
	n += 1
	copy(arr[n:], []byte(fmt.Sprintf("%02d", p.cost)))
	n += 2
	arr[n] = '$'
	n += 1
	copy(arr[n:], p.salt)
	n += encodedSaltSize
	copy(arr[n:], p.hash)
	n += encodedHashSize
	return arr[:n]
}

func (p *hashed) decodeVersion(sbytes []byte) (int, error) {
	if sbytes[0] != '$' {
		return -1, InvalidHashPrefixError(sbytes[0])
	}
	if sbytes[1] > majorVersion {
		return -1, HashVersionTooNewError(sbytes[1])
	}
	p.major = sbytes[1]
	n := 3
	if sbytes[2] != '$' {
		p.minor = sbytes[2]
		n++
	}
	return n, nil
}

// sbytes should begin where decodeVersion left off.
func (p *hashed) decodeCost(sbytes []byte) (int, error) {
	cost, err := strconv.Atoi(string(sbytes[0:2]))
	if err != nil {
		return -1, err
	}
	err = checkCost(cost)
	if err != nil {
		return -1, err
	}
	p.cost = uint32(cost)
	return 3, nil
}

func (p *hashed) String() string {
	return fmt.Sprintf("&{hash: %#v, salt: %#v, cost: %d, major: %c, minor: %c}", string(p.hash), p.salt, p.cost, p.major, p.minor)
}

func checkCost(cost int) error {
	if cost < MinCost || cost > MaxCost {
		return InvalidCostError(cost)
	}
	return nil
}
<commit_msg>bcrypt: Correct typo in package comment.<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package bcrypt implements Provos and Mazières's bcrypt adaptive hashing
// algorithm. See http://www.usenix.org/event/usenix99/provos/provos.pdf
package bcrypt

// The code is a port of Provos and Mazières's C implementation.
import (
	"crypto/blowfish"
	"crypto/rand"
	"crypto/subtle"
	"errors"
	"fmt"
	"io"
	"strconv"
)

const (
	MinCost     int = 4  // the minimum allowable cost as passed in to GenerateFromPassword
	MaxCost     int = 31 // the maximum allowable cost as passed in to GenerateFromPassword
	DefaultCost int = 10 // the cost that will actually be set if a cost below MinCost is passed into GenerateFromPassword
)

// The error returned from CompareHashAndPassword when a password and hash do
// not match.
var MismatchedHashAndPasswordError = errors.New("crypto/bcrypt: hashedPassword is not the hash of the given password")

// The error returned from CompareHashAndPassword when a hash is too short to
// be a bcrypt hash.
var HashTooShortError = errors.New("crypto/bcrypt: hashedSecret too short to be a bcrypted password")

// The error returned from CompareHashAndPassword when a hash was created with
// a bcrypt algorithm newer than this implementation.
type HashVersionTooNewError byte

func (hv HashVersionTooNewError) Error() string {
	return fmt.Sprintf("crypto/bcrypt: bcrypt algorithm version '%c' requested is newer than current version '%c'", byte(hv), majorVersion)
}

// The error returned from CompareHashAndPassword when a hash starts with something other than '$'
type InvalidHashPrefixError byte

func (ih InvalidHashPrefixError) Error() string {
	return fmt.Sprintf("crypto/bcrypt: bcrypt hashes must start with '$', but hashedSecret started with '%c'", byte(ih))
}

type InvalidCostError int

func (ic InvalidCostError) Error() string {
	return fmt.Sprintf("crypto/bcrypt: cost %d is outside allowed range (%d,%d)", int(ic), int(MinCost), int(MaxCost))
}

const (
	majorVersion       = '2'
	minorVersion       = 'a'
	maxSaltSize        = 16
	maxCryptedHashSize = 23
	encodedSaltSize    = 22
	encodedHashSize    = 31
	minHashSize        = 59
)

// magicCipherData is an IV for the 64 Blowfish encryption calls in
// bcrypt(). It's the string "OrpheanBeholderScryDoubt" in big-endian bytes.
var magicCipherData = []byte{
	0x4f, 0x72, 0x70, 0x68,
	0x65, 0x61, 0x6e, 0x42,
	0x65, 0x68, 0x6f, 0x6c,
	0x64, 0x65, 0x72, 0x53,
	0x63, 0x72, 0x79, 0x44,
	0x6f, 0x75, 0x62, 0x74,
}

type hashed struct {
	hash  []byte
	salt  []byte
	cost  uint32 // allowed range is MinCost to MaxCost
	major byte
	minor byte
}

// GenerateFromPassword returns the bcrypt hash of the password at the given
// cost. If the cost given is less than MinCost, the cost will be set to
// MinCost, instead. Use CompareHashAndPassword, as defined in this package,
// to compare the returned hashed password with its cleartext version.
func GenerateFromPassword(password []byte, cost int) ([]byte, error) {
	p, err := newFromPassword(password, cost)
	if err != nil {
		return nil, err
	}
	return p.Hash(), nil
}

// CompareHashAndPassword compares a bcrypt hashed password with its possible
// plaintext equivalent. Note: Using bytes.Equal for this job is
// insecure. Returns nil on success, or an error on failure.
func CompareHashAndPassword(hashedPassword, password []byte) error {
	p, err := newFromHash(hashedPassword)
	if err != nil {
		return err
	}

	otherHash, err := bcrypt(password, p.cost, p.salt)
	if err != nil {
		return err
	}

	otherP := &hashed{otherHash, p.salt, p.cost, p.major, p.minor}
	if subtle.ConstantTimeCompare(p.Hash(), otherP.Hash()) == 1 {
		return nil
	}

	return MismatchedHashAndPasswordError
}

func newFromPassword(password []byte, cost int) (*hashed, error) {
	if cost < MinCost {
		cost = DefaultCost
	}
	p := new(hashed)
	p.major = majorVersion
	p.minor = minorVersion

	err := checkCost(cost)
	if err != nil {
		return nil, err
	}
	p.cost = uint32(cost)

	unencodedSalt := make([]byte, maxSaltSize)
	_, err = io.ReadFull(rand.Reader, unencodedSalt)
	if err != nil {
		return nil, err
	}

	p.salt = base64Encode(unencodedSalt)
	hash, err := bcrypt(password, p.cost, p.salt)
	if err != nil {
		return nil, err
	}
	p.hash = hash
	return p, err
}

func newFromHash(hashedSecret []byte) (*hashed, error) {
	if len(hashedSecret) < minHashSize {
		return nil, HashTooShortError
	}
	p := new(hashed)
	n, err := p.decodeVersion(hashedSecret)
	if err != nil {
		return nil, err
	}
	hashedSecret = hashedSecret[n:]
	n, err = p.decodeCost(hashedSecret)
	if err != nil {
		return nil, err
	}
	hashedSecret = hashedSecret[n:]

	// The "+2" is here because we'll have to append at most 2 '=' to the salt
	// when base64 decoding it in expensiveBlowfishSetup().
	p.salt = make([]byte, encodedSaltSize, encodedSaltSize+2)
	copy(p.salt, hashedSecret[:encodedSaltSize])

	hashedSecret = hashedSecret[encodedSaltSize:]
	p.hash = make([]byte, len(hashedSecret))
	copy(p.hash, hashedSecret)

	return p, nil
}

func bcrypt(password []byte, cost uint32, salt []byte) ([]byte, error) {
	cipherData := make([]byte, len(magicCipherData))
	copy(cipherData, magicCipherData)

	c, err := expensiveBlowfishSetup(password, cost, salt)
	if err != nil {
		return nil, err
	}

	for i := 0; i < 24; i += 8 {
		for j := 0; j < 64; j++ {
			c.Encrypt(cipherData[i:i+8], cipherData[i:i+8])
		}
	}

	// Bug compatibility with C bcrypt implementations. We only encode 23 of
	// the 24 bytes encrypted.
	hsh := base64Encode(cipherData[:maxCryptedHashSize])
	return hsh, nil
}

func expensiveBlowfishSetup(key []byte, cost uint32, salt []byte) (*blowfish.Cipher, error) {

	csalt, err := base64Decode(salt)
	if err != nil {
		return nil, err
	}

	// Bug compatibility with C bcrypt implementations. They use the trailing
	// NULL in the key string during expansion.
	ckey := append(key, 0)

	c, err := blowfish.NewSaltedCipher(ckey, csalt)
	if err != nil {
		return nil, err
	}

	rounds := 1 << cost
	for i := 0; i < rounds; i++ {
		blowfish.ExpandKey(ckey, c)
		blowfish.ExpandKey(csalt, c)
	}

	return c, nil
}

func (p *hashed) Hash() []byte {
	arr := make([]byte, 60)
	arr[0] = '$'
	arr[1] = p.major
	n := 2
	if p.minor != 0 {
		arr[2] = p.minor
		n = 3
	}
	arr[n] = '$'
	n += 1
	copy(arr[n:], []byte(fmt.Sprintf("%02d", p.cost)))
	n += 2
	arr[n] = '$'
	n += 1
	copy(arr[n:], p.salt)
	n += encodedSaltSize
	copy(arr[n:], p.hash)
	n += encodedHashSize
	return arr[:n]
}

func (p *hashed) decodeVersion(sbytes []byte) (int, error) {
	if sbytes[0] != '$' {
		return -1, InvalidHashPrefixError(sbytes[0])
	}
	if sbytes[1] > majorVersion {
		return -1, HashVersionTooNewError(sbytes[1])
	}
	p.major = sbytes[1]
	n := 3
	if sbytes[2] != '$' {
		p.minor = sbytes[2]
		n++
	}
	return n, nil
}

// sbytes should begin where decodeVersion left off.
func (p *hashed) decodeCost(sbytes []byte) (int, error) {
	cost, err := strconv.Atoi(string(sbytes[0:2]))
	if err != nil {
		return -1, err
	}
	err = checkCost(cost)
	if err != nil {
		return -1, err
	}
	p.cost = uint32(cost)
	return 3, nil
}

func (p *hashed) String() string {
	return fmt.Sprintf("&{hash: %#v, salt: %#v, cost: %d, major: %c, minor: %c}", string(p.hash), p.salt, p.cost, p.major, p.minor)
}

func checkCost(cost int) error {
	if cost < MinCost || cost > MaxCost {
		return InvalidCostError(cost)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"github.com/mvdan/xurls"
	"github.com/paddycarey/gophy"
	"log"
	"net/url"
	"strconv"
)

func (theLobby *lobby) sendGiphy(searchTerm string, authorName string, userId int64, messageId int64) {
	searchTerm = url.QueryEscape(searchTerm)
	gophyOptions := &gophy.ClientOptions{}
	gophyClient := gophy.NewClient(gophyOptions)

	// Search for the particular gif with the parameters
	// "" -> rating (.e.g PG, PG-13, etc)... We want it all
	// 1 is the number of entries, we just want one
	// 0 is the offet, how many pages
	gifs, num, err := gophyClient.SearchGifs(searchTerm, "", 1, 0)
	if err != nil {
		log.Println("error searching giphy: ", err)
		return
	}
	if num > 0 {
		imageUrl := gifs[0].Images.FixedWidth.URL
		giphyMessage := `<img src="` + imageUrl + `" alt="` + searchTerm + `" title="` + searchTerm + `">`
		theLobby.broadcast <- &internalMessage{
			MessageText:        []byte(giphyMessage),
			MessageDisplayName: []byte(authorName),
			MessageAuthorId:    userId,
			MessageId:          messageId,
		}
	}
}

func (theLobby *lobby) sendStar(messageToStar string, starrerId int64, starringMessageId int64) {
	// Find the real message id, versus the channel message id... It gets confusing
	rows, err := db.Query("SELECT * FROM messages WHERE channel_msg_id = ? AND channel_id = ?", messageToStar, theLobby.channelId)
	if err != nil {
		log.Println("Error finding messages for sendStar: ", err)
		return
	}

	// Read in the singular message in this scope
	var message_to_star_real_id int64
	var channel_id int64
	var message_text string
	var channel_msg_id int64
	var author_display_name string
	var author_id int64

	foundOne := false
	for rows.Next() {
		foundOne = true
		err = rows.Scan(&message_to_star_real_id, &channel_id, &message_text, &channel_msg_id, &author_display_name, &author_id)
		if err != nil {
			log.Println("error scanning row", err)
			return
		}
		break
	}
	rows.Close()

	if !foundOne {
		// We don't actually have a message... Let's back out
		return
	}

	// Now that we have the true message id, insert our star
	_, err = db.Exec("INSERT INTO stars(id, starrer_id, starree_id, message_id) VALUES (?, ?, ?, ?);", nil, starrerId, author_id, message_to_star_real_id)
	if err != nil {
		log.Println("error writing star to database", err)
		return
	}

	// Now, get all the stars
	stars, err := db.Query("SELECT * FROM stars WHERE message_id = ?;", message_to_star_real_id)
	if err != nil {
		log.Println("error loading stars", err)
		return
	}
	defer stars.Close()
	var starCounter = 0

	// Count the stars
	for stars.Next() {
		var star_id, starrer_id, starree_id, message_id int64
		stars.Scan(&star_id, &starrer_id, &starree_id, &message_id)
		starCounter += 1
	}

	// Special message to send to the front end ...
	type StarMessage struct {
		MessageId int64
		NumStars  int
	}

	i, err := strconv.ParseInt(messageToStar, 10, 64)
	if err != nil {
		log.Println("couldnt parse string into an integer", err)
		return
	}

	outgoingMessage := StarMessage{
		MessageId: i,
		NumStars:  starCounter,
	}

	marshalled, err := json.Marshal(outgoingMessage)
	if err != nil {
		log.Println("error marshalling star message", err)
		return
	}

	// ... from a magic sender
	// TODO: Make it so normal users cannot send with the user __ADMIN__
	theLobby.broadcast <- &internalMessage{
		MessageText:        marshalled,
		MessageDisplayName: []byte("__ADMIN__"),
		MessageAuthorId:    0,
		MessageId:          starringMessageId,
	}

}

func (theLobby *lobby) linkifyMessage(messageString string, messageAuthorId int64, messageDisplayName string, messageId int64) {

	linkifiedMessage := xurls.Relaxed.ReplaceAllStringFunc(messageString, func(inString string) string {
		url, err := url.Parse(inString)
		var scheme string
		if err == nil {
			scheme = url.Scheme
		} else {
			log.Println("error parsing url: ", err)
		}
		if scheme == "" {
			url.Scheme = "http"
		}
		return `<a href="` + url.String() + `">` + inString + `</a>`
	})

	// Construct an internal struct, this case including our internal user id
	outgoingMessage := &internalMessage{MessageText: []byte(linkifiedMessage), MessageAuthorId: messageAuthorId, MessageDisplayName: []byte(messageDisplayName), MessageId: messageId}

	// Send the message out for broadcast
	theLobby.broadcast <- outgoingMessage

}
<commit_msg>Add a class to gif images for front end<commit_after>package main

import (
	"encoding/json"
	"github.com/mvdan/xurls"
	"github.com/paddycarey/gophy"
	"log"
	"net/url"
	"strconv"
)

func (theLobby *lobby) sendGiphy(searchTerm string, authorName string, userId int64, messageId int64) {
	searchTerm = url.QueryEscape(searchTerm)
	gophyOptions := &gophy.ClientOptions{}
	gophyClient := gophy.NewClient(gophyOptions)

	// Search for the particular gif with the parameters
	// "" -> rating (.e.g PG, PG-13, etc)... We want it all
	// 1 is the number of entries, we just want one
	// 0 is the offet, how many pages
	gifs, num, err := gophyClient.SearchGifs(searchTerm, "", 1, 0)
	if err != nil {
		log.Println("error searching giphy: ", err)
		return
	}
	if num > 0 {
		imageUrl := gifs[0].Images.FixedWidth.URL
		giphyMessage := `<img src="` + imageUrl + `" alt="` + searchTerm + `" title="` + searchTerm + `" class="gif" >`
		theLobby.broadcast <- &internalMessage{
			MessageText:        []byte(giphyMessage),
			MessageDisplayName: []byte(authorName),
			MessageAuthorId:    userId,
			MessageId:          messageId,
		}
	}
}

func (theLobby *lobby) sendStar(messageToStar string, starrerId int64, starringMessageId int64) {
	// Find the real message id, versus the channel message id... It gets confusing
	rows, err := db.Query("SELECT * FROM messages WHERE channel_msg_id = ? AND channel_id = ?", messageToStar, theLobby.channelId)
	if err != nil {
		log.Println("Error finding messages for sendStar: ", err)
		return
	}

	// Read in the singular message in this scope
	var message_to_star_real_id int64
	var channel_id int64
	var message_text string
	var channel_msg_id int64
	var author_display_name string
	var author_id int64

	foundOne := false
	for rows.Next() {
		foundOne = true
		err = rows.Scan(&message_to_star_real_id, &channel_id, &message_text, &channel_msg_id, &author_display_name, &author_id)
		if err != nil {
			log.Println("error scanning row", err)
			return
		}
		break
	}
	rows.Close()

	if !foundOne {
		// We don't actually have a message... Let's back out
		return
	}

	// Now that we have the true message id, insert our star
	_, err = db.Exec("INSERT INTO stars(id, starrer_id, starree_id, message_id) VALUES (?, ?, ?, ?);", nil, starrerId, author_id, message_to_star_real_id)
	if err != nil {
		log.Println("error writing star to database", err)
		return
	}

	// Now, get all the stars
	stars, err := db.Query("SELECT * FROM stars WHERE message_id = ?;", message_to_star_real_id)
	if err != nil {
		log.Println("error loading stars", err)
		return
	}
	defer stars.Close()
	var starCounter = 0

	// Count the stars
	for stars.Next() {
		var star_id, starrer_id, starree_id, message_id int64
		stars.Scan(&star_id, &starrer_id, &starree_id, &message_id)
		starCounter += 1
	}

	// Special message to send to the front end ...
	type StarMessage struct {
		MessageId int64
		NumStars  int
	}

	i, err := strconv.ParseInt(messageToStar, 10, 64)
	if err != nil {
		log.Println("couldnt parse string into an integer", err)
		return
	}

	outgoingMessage := StarMessage{
		MessageId: i,
		NumStars:  starCounter,
	}

	marshalled, err := json.Marshal(outgoingMessage)
	if err != nil {
		log.Println("error marshalling star message", err)
		return
	}

	// ... from a magic sender
	// TODO: Make it so normal users cannot send with the user __ADMIN__
	theLobby.broadcast <- &internalMessage{
		MessageText:        marshalled,
		MessageDisplayName: []byte("__ADMIN__"),
		MessageAuthorId:    0,
		MessageId:          starringMessageId,
	}

}

func (theLobby *lobby) linkifyMessage(messageString string, messageAuthorId int64, messageDisplayName string, messageId int64) {

	linkifiedMessage := xurls.Relaxed.ReplaceAllStringFunc(messageString, func(inString string) string {
		url, err := url.Parse(inString)
		var scheme string
		if err == nil {
			scheme = url.Scheme
		} else {
			log.Println("error parsing url: ", err)
		}
		if scheme == "" {
			url.Scheme = "http"
		}
		return `<a href="` + url.String() + `">` + inString + `</a>`
	})

	// Construct an internal struct, this case including our internal user id
	outgoingMessage := &internalMessage{MessageText: []byte(linkifiedMessage), MessageAuthorId: messageAuthorId, MessageDisplayName: []byte(messageDisplayName), MessageId: messageId}

	// Send the message out for broadcast
	theLobby.broadcast <- outgoingMessage

}
<|endoftext|>


<commit_before>package linkedql

import (
	"encoding/json"
	"fmt"
	"reflect"
	"unicode"
	"unicode/utf8"

	"github.com/cayleygraph/quad"
)

var valueStep = reflect.TypeOf((*ValueStep)(nil)).Elem()
var step = reflect.TypeOf((*Step)(nil)).Elem()

func typeToRange(t reflect.Type) interface{} {
	if t.Kind() == reflect.Slice {
		return typeToRange(t.Elem())
	}
	if t.Kind() == reflect.String {
		return map[string]interface{}{"@id": "xsd:string"}
	}
	if t.Kind() == reflect.Bool {
		return map[string]interface{}{"@id": "xsd:bool"}
	}
	if kind := t.Kind(); kind == reflect.Int64 || kind == reflect.Int {
		return map[string]interface{}{"@id": "xsd:int"}
	}
	if t.Implements(valueStep) {
		return map[string]interface{}{"@id": "linkedql:ValueStep"}
	}
	if t.Implements(step) {
		return map[string]interface{}{"@id": "linkedql:Step"}
	}
	if t.Implements(reflect.TypeOf((*Operator)(nil)).Elem()) {
		return map[string]interface{}{"@id": "linkedql:Operator"}
	}
	if t.Implements(reflect.TypeOf((*quad.Value)(nil)).Elem()) {
		return map[string]interface{}{"@id": "rdfs:Resource"}
	}
	fmt.Printf("%v", t)
	panic("Unexpected type")
}

func lcFirst(str string) string {
	rune, size := utf8.DecodeRuneInString(str)
	return string(unicode.ToLower(rune)) + str[size:]
}

func typeToDocuments(name string, t reflect.Type) []interface{} {
	var documents []interface{}
	var superClasses []string
	if t.Implements(valueStep) {
		superClasses = append(superClasses, "linkedql:ValueStep")
	} else {
		superClasses = append(superClasses, "linkedql:Step")
	}
	documents = append(documents, map[string]interface{}{
		"@id":             name,
		"@type":           "rdfs:Class",
		"rdfs:subClassOf": superClasses,
	})
	for i := 0; i < t.NumField(); i++ {
		f := t.Field(i)
		property := "linkedql:" + lcFirst(f.Name)
		if f.Type.Kind() != reflect.Slice {
			restriction := quad.RandomBlankNode().String()
			superClasses = append(superClasses, restriction)
			documents = append(documents, map[string]interface{}{
				"@id":             restriction,
				"@type":           "owl:Restriction",
				"owl:cardinality": 1,
				"owl:onProperty":  map[string]interface{}{"@id": property},
			})
		}
		documents = append(documents, map[string]interface{}{
			// TODO(iddan): use json tag instead
			"@id":         property,
			"@type":       "rdf:Property",
			"rdfs:domain": map[string]interface{}{"@id": name},
			"rdfs:range":  typeToRange(f.Type),
		})
	}
	return documents
}

func generateSchema() []interface{} {
	var documents []interface{}
	for name, _type := range typeByName {
		for _, document := range typeToDocuments(name, _type) {
			documents = append(documents, document)
		}
	}
	return documents
}

func serializeSchema() []byte {
	bytes, err := json.MarshalIndent(generateSchema(), "", "    ")
	if err != nil {
		panic(err)
	}
	return bytes
}
<commit_msg>linkedql: Fix schema not registered<commit_after>package linkedql

import (
	"encoding/json"
	"fmt"
	"reflect"
	"unicode"
	"unicode/utf8"

	"github.com/cayleygraph/quad"
)

var valueStep = reflect.TypeOf((*ValueStep)(nil)).Elem()
var step = reflect.TypeOf((*Step)(nil)).Elem()

func typeToRange(t reflect.Type) interface{} {
	if t.Kind() == reflect.Slice {
		return typeToRange(t.Elem())
	}
	if t.Kind() == reflect.String {
		return map[string]interface{}{"@id": "xsd:string"}
	}
	if t.Kind() == reflect.Bool {
		return map[string]interface{}{"@id": "xsd:bool"}
	}
	if kind := t.Kind(); kind == reflect.Int64 || kind == reflect.Int {
		return map[string]interface{}{"@id": "xsd:int"}
	}
	if t.Implements(valueStep) {
		return map[string]interface{}{"@id": "linkedql:ValueStep"}
	}
	if t.Implements(step) {
		return map[string]interface{}{"@id": "linkedql:Step"}
	}
	if t.Implements(reflect.TypeOf((*Operator)(nil)).Elem()) {
		return map[string]interface{}{"@id": "linkedql:Operator"}
	}
	if t.Implements(reflect.TypeOf((*quad.Value)(nil)).Elem()) {
		return map[string]interface{}{"@id": "rdfs:Resource"}
	}
	fmt.Printf("%v", t)
	panic("Unexpected type")
}

func lcFirst(str string) string {
	rune, size := utf8.DecodeRuneInString(str)
	return string(unicode.ToLower(rune)) + str[size:]
}

func typeToDocuments(name string, t reflect.Type) []interface{} {
	var documents []interface{}
	var superClasses []string
	if t.Implements(valueStep) {
		superClasses = append(superClasses, "linkedql:ValueStep")
	} else {
		superClasses = append(superClasses, "linkedql:Step")
	}
	for i := 0; i < t.NumField(); i++ {
		f := t.Field(i)
		property := "linkedql:" + lcFirst(f.Name)
		if f.Type.Kind() != reflect.Slice {
			restriction := quad.RandomBlankNode().String()
			superClasses = append(superClasses, restriction)
			documents = append(documents, map[string]interface{}{
				"@id":             restriction,
				"@type":           "owl:Restriction",
				"owl:cardinality": 1,
				"owl:onProperty":  map[string]interface{}{"@id": property},
			})
		}
		documents = append(documents, map[string]interface{}{
			// TODO(iddan): use json tag instead
			"@id":         property,
			"@type":       "rdf:Property",
			"rdfs:domain": map[string]interface{}{"@id": name},
			"rdfs:range":  typeToRange(f.Type),
		})
	}
	documents = append(documents, map[string]interface{}{
		"@id":             name,
		"@type":           "rdfs:Class",
		"rdfs:subClassOf": superClasses,
	})
	return documents
}

func generateSchema() []interface{} {
	var documents []interface{}
	for name, _type := range typeByName {
		for _, document := range typeToDocuments(name, _type) {
			documents = append(documents, document)
		}
	}
	return documents
}

func serializeSchema() []byte {
	bytes, err := json.MarshalIndent(generateSchema(), "", "    ")
	if err != nil {
		panic(err)
	}
	return bytes
}
<|endoftext|>