<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package testsuites

import (
	"fmt"
	"strconv"

	"github.com/onsi/ginkgo"
	v1 "k8s.io/api/core/v1"
	errors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/kubernetes/test/e2e/framework"
	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
	e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
	storageframework "k8s.io/kubernetes/test/e2e/storage/framework"
	storageutils "k8s.io/kubernetes/test/e2e/storage/utils"
	utilpointer "k8s.io/utils/pointer"
)

const (
	rootDir         = "/mnt/volume1"
	rootDirFile     = "file1"
	rootDirFilePath = rootDir + "/" + rootDirFile
	subdir          = "/mnt/volume1/subdir"
	subDirFile      = "file2"
	subDirFilePath  = subdir + "/" + subDirFile
)

type fsGroupChangePolicyTestSuite struct {
	tsInfo storageframework.TestSuiteInfo
}

var _ storageframework.TestSuite = &fsGroupChangePolicyTestSuite{}

// InitCustomFsGroupChangePolicyTestSuite returns fsGroupChangePolicyTestSuite that implements TestSuite interface
func InitCustomFsGroupChangePolicyTestSuite(patterns []storageframework.TestPattern) storageframework.TestSuite {
	return &fsGroupChangePolicyTestSuite{
		tsInfo: storageframework.TestSuiteInfo{
			Name:         "fsgroupchangepolicy",
			TestPatterns: patterns,
			SupportedSizeRange: e2evolume.SizeRange{
				Min: "1Mi",
			},
		},
	}
}

// InitFsGroupChangePolicyTestSuite returns fsGroupChangePolicyTestSuite that implements TestSuite interface
func InitFsGroupChangePolicyTestSuite() storageframework.TestSuite {
	patterns := []storageframework.TestPattern{
		storageframework.DefaultFsDynamicPV,
	}
	return InitCustomFsGroupChangePolicyTestSuite(patterns)
}

func (s *fsGroupChangePolicyTestSuite) GetTestSuiteInfo() storageframework.TestSuiteInfo {
	return s.tsInfo
}

func (s *fsGroupChangePolicyTestSuite) SkipUnsupportedTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) {
	skipVolTypePatterns(pattern, driver, storageframework.NewVolTypeMap(storageframework.CSIInlineVolume, storageframework.GenericEphemeralVolume))
	dInfo := driver.GetDriverInfo()
	if !dInfo.Capabilities[storageframework.CapFsGroup] {
		e2eskipper.Skipf("Driver %q does not support FsGroup - skipping", dInfo.Name)
	}

	if pattern.VolMode == v1.PersistentVolumeBlock {
		e2eskipper.Skipf("Test does not support non-filesystem volume mode - skipping")
	}

	if pattern.VolType != storageframework.DynamicPV {
		e2eskipper.Skipf("Suite %q does not support %v", s.tsInfo.Name, pattern.VolType)
	}

	_, ok := driver.(storageframework.DynamicPVTestDriver)
	if !ok {
		e2eskipper.Skipf("Driver %s doesn't support %v -- skipping", dInfo.Name, pattern.VolType)
	}
}

func (s *fsGroupChangePolicyTestSuite) DefineTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) {
	type local struct {
		config        *storageframework.PerTestConfig
		driverCleanup func()
		driver        storageframework.TestDriver
		resource      *storageframework.VolumeResource
	}
	var l local

	// Beware that it also registers an AfterEach which renders f unusable. Any code using
	// f must run inside an It or Context callback.
	f := framework.NewFrameworkWithCustomTimeouts("fsgroupchangepolicy", storageframework.GetDriverTimeouts(driver))

	init := func() {
		e2eskipper.SkipIfNodeOSDistroIs("windows")
		l = local{}
		l.driver = driver
		l.config, l.driverCleanup = driver.PrepareTest(f)
		testVolumeSizeRange := s.GetTestSuiteInfo().SupportedSizeRange
		l.resource = storageframework.CreateVolumeResource(l.driver, l.config, pattern, testVolumeSizeRange)
	}

	cleanup := func() {
		var errs []error
		if l.resource != nil {
			if err := l.resource.CleanupResource(); err != nil {
				errs = append(errs, err)
			}
			l.resource = nil
		}

		if l.driverCleanup != nil {
			errs = append(errs, storageutils.TryFunc(l.driverCleanup))
			l.driverCleanup = nil
		}

		framework.ExpectNoError(errors.NewAggregate(errs), "while cleanup resource")
	}

	tests := []struct {
		name                              string // Test case name
		podfsGroupChangePolicy            string // 'Always' or 'OnRootMismatch'
		initialPodFsGroup                 int    // FsGroup of the initial pod
		changedRootDirFileOwnership       int    // Change the ownership of the file in the root directory (/mnt/volume1/file1), as part of the initial pod
		changedSubDirFileOwnership        int    // Change the ownership of the file in the sub directory (/mnt/volume1/subdir/file2), as part of the initial pod
		secondPodFsGroup                  int    // FsGroup of the second pod
		finalExpectedRootDirFileOwnership int    // Final expcted ownership of the file in the root directory (/mnt/volume1/file1), as part of the second pod
		finalExpectedSubDirFileOwnership  int    // Final expcted ownership of the file in the sub directory (/mnt/volume1/subdir/file2), as part of the second pod
	}{
		// Test cases for 'Always' policy
		{
			name:                              "pod created with an initial fsgroup, new pod fsgroup applied to volume contents",
			podfsGroupChangePolicy:            "Always",
			initialPodFsGroup:                 1000,
			secondPodFsGroup:                  2000,
			finalExpectedRootDirFileOwnership: 2000,
			finalExpectedSubDirFileOwnership:  2000,
		},
		{
			name:                              "pod created with an initial fsgroup, volume contents ownership changed in first pod, new pod with same fsgroup applied to the volume contents",
			podfsGroupChangePolicy:            "Always",
			initialPodFsGroup:                 1000,
			changedRootDirFileOwnership:       2000,
			changedSubDirFileOwnership:        3000,
			secondPodFsGroup:                  1000,
			finalExpectedRootDirFileOwnership: 1000,
			finalExpectedSubDirFileOwnership:  1000,
		},
		{
			name:                              "pod created with an initial fsgroup, volume contents ownership changed in first pod, new pod with different fsgroup applied to the volume contents",
			podfsGroupChangePolicy:            "Always",
			initialPodFsGroup:                 1000,
			changedRootDirFileOwnership:       2000,
			changedSubDirFileOwnership:        3000,
			secondPodFsGroup:                  4000,
			finalExpectedRootDirFileOwnership: 4000,
			finalExpectedSubDirFileOwnership:  4000,
		},
		// Test cases for 'OnRootMismatch' policy
		{
			name:                              "pod created with an initial fsgroup, new pod fsgroup applied to volume contents",
			podfsGroupChangePolicy:            "OnRootMismatch",
			initialPodFsGroup:                 1000,
			secondPodFsGroup:                  2000,
			finalExpectedRootDirFileOwnership: 2000,
			finalExpectedSubDirFileOwnership:  2000,
		},
		{
			name:                              "pod created with an initial fsgroup, volume contents ownership changed in first pod, new pod with same fsgroup skips ownership changes to the volume contents",
			podfsGroupChangePolicy:            "OnRootMismatch",
			initialPodFsGroup:                 1000,
			changedRootDirFileOwnership:       2000,
			changedSubDirFileOwnership:        3000,
			secondPodFsGroup:                  1000,
			finalExpectedRootDirFileOwnership: 2000,
			finalExpectedSubDirFileOwnership:  3000,
		},
		{
			name:                              "pod created with an initial fsgroup, volume contents ownership changed in first pod, new pod with different fsgroup applied to the volume contents",
			podfsGroupChangePolicy:            "OnRootMismatch",
			initialPodFsGroup:                 1000,
			changedRootDirFileOwnership:       2000,
			changedSubDirFileOwnership:        3000,
			secondPodFsGroup:                  4000,
			finalExpectedRootDirFileOwnership: 4000,
			finalExpectedSubDirFileOwnership:  4000,
		},
	}

	for _, t := range tests {
		test := t
		testCaseName := fmt.Sprintf("(%s)[LinuxOnly], %s", test.podfsGroupChangePolicy, test.name)
		ginkgo.It(testCaseName, func() {
			dInfo := driver.GetDriverInfo()
			policy := v1.PodFSGroupChangePolicy(test.podfsGroupChangePolicy)

			if dInfo.Capabilities[storageframework.CapVolumeMountGroup] &&
				policy == v1.FSGroupChangeOnRootMismatch {
				e2eskipper.Skipf("Driver %q supports VolumeMountGroup, which doesn't supported the OnRootMismatch FSGroup policy - skipping", dInfo.Name)
			}

			init()
			defer cleanup()
			podConfig := e2epod.Config{
				NS:                     f.Namespace.Name,
				NodeSelection:          l.config.ClientNodeSelection,
				PVCs:                   []*v1.PersistentVolumeClaim{l.resource.Pvc},
				FsGroup:                utilpointer.Int64Ptr(int64(test.initialPodFsGroup)),
				PodFSGroupChangePolicy: &policy,
			}
			// Create initial pod and create files in root and sub-directory and verify ownership.
			pod := createPodAndVerifyContentGid(l.config.Framework, &podConfig, true /* createInitialFiles */, "" /* expectedRootDirFileOwnership */, "" /* expectedSubDirFileOwnership */)

			// Change the ownership of files in the initial pod.
			if test.changedRootDirFileOwnership != 0 {
				ginkgo.By(fmt.Sprintf("Changing the root directory file ownership to %s", strconv.Itoa(test.changedRootDirFileOwnership)))
				storageutils.ChangeFilePathGidInPod(f, rootDirFilePath, strconv.Itoa(test.changedRootDirFileOwnership), pod)
			}

			if test.changedSubDirFileOwnership != 0 {
				ginkgo.By(fmt.Sprintf("Changing the sub-directory file ownership to %s", strconv.Itoa(test.changedSubDirFileOwnership)))
				storageutils.ChangeFilePathGidInPod(f, subDirFilePath, strconv.Itoa(test.changedSubDirFileOwnership), pod)
			}

			ginkgo.By(fmt.Sprintf("Deleting Pod %s/%s", pod.Namespace, pod.Name))
			framework.ExpectNoError(e2epod.DeletePodWithWait(f.ClientSet, pod))

			// Create a second pod with existing volume and verify the contents ownership.
			podConfig.FsGroup = utilpointer.Int64Ptr(int64(test.secondPodFsGroup))
			pod = createPodAndVerifyContentGid(l.config.Framework, &podConfig, false /* createInitialFiles */, strconv.Itoa(test.finalExpectedRootDirFileOwnership), strconv.Itoa(test.finalExpectedSubDirFileOwnership))
			ginkgo.By(fmt.Sprintf("Deleting Pod %s/%s", pod.Namespace, pod.Name))
			framework.ExpectNoError(e2epod.DeletePodWithWait(f.ClientSet, pod))
		})
	}
}

func createPodAndVerifyContentGid(f *framework.Framework, podConfig *e2epod.Config, createInitialFiles bool, expectedRootDirFileOwnership, expectedSubDirFileOwnership string) *v1.Pod {
	podFsGroup := strconv.FormatInt(*podConfig.FsGroup, 10)
	ginkgo.By(fmt.Sprintf("Creating Pod in namespace %s with fsgroup %s", podConfig.NS, podFsGroup))
	pod, err := e2epod.CreateSecPodWithNodeSelection(f.ClientSet, podConfig, f.Timeouts.PodStart)
	framework.ExpectNoError(err)
	framework.Logf("Pod %s/%s started successfully", pod.Namespace, pod.Name)

	if createInitialFiles {
		ginkgo.By(fmt.Sprintf("Creating a sub-directory and file, and verifying their ownership is %s", podFsGroup))
		cmd := fmt.Sprintf("touch %s", rootDirFilePath)
		var err error
		_, _, err = e2evolume.PodExec(f, pod, cmd)
		framework.ExpectNoError(err)
		storageutils.VerifyFilePathGidInPod(f, rootDirFilePath, podFsGroup, pod)

		cmd = fmt.Sprintf("mkdir %s", subdir)
		_, _, err = e2evolume.PodExec(f, pod, cmd)
		framework.ExpectNoError(err)
		cmd = fmt.Sprintf("touch %s", subDirFilePath)
		_, _, err = e2evolume.PodExec(f, pod, cmd)
		framework.ExpectNoError(err)
		storageutils.VerifyFilePathGidInPod(f, subDirFilePath, podFsGroup, pod)
		return pod
	}

	// Verify existing contents of the volume
	ginkgo.By(fmt.Sprintf("Verifying the ownership of root directory file is %s", expectedRootDirFileOwnership))
	storageutils.VerifyFilePathGidInPod(f, rootDirFilePath, expectedRootDirFileOwnership, pod)
	ginkgo.By(fmt.Sprintf("Verifying the ownership of sub directory file is %s", expectedSubDirFileOwnership))
	storageutils.VerifyFilePathGidInPod(f, subDirFilePath, expectedSubDirFileOwnership, pod)
	return pod
}
<commit_msg>DelegateFSGroupToCSIDriver e2e: skip tests with chgrp<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package testsuites

import (
	"fmt"
	"strconv"

	"github.com/onsi/ginkgo"
	v1 "k8s.io/api/core/v1"
	errors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/kubernetes/test/e2e/framework"
	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
	e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
	storageframework "k8s.io/kubernetes/test/e2e/storage/framework"
	storageutils "k8s.io/kubernetes/test/e2e/storage/utils"
	utilpointer "k8s.io/utils/pointer"
)

const (
	rootDir         = "/mnt/volume1"
	rootDirFile     = "file1"
	rootDirFilePath = rootDir + "/" + rootDirFile
	subdir          = "/mnt/volume1/subdir"
	subDirFile      = "file2"
	subDirFilePath  = subdir + "/" + subDirFile
)

type fsGroupChangePolicyTestSuite struct {
	tsInfo storageframework.TestSuiteInfo
}

var _ storageframework.TestSuite = &fsGroupChangePolicyTestSuite{}

// InitCustomFsGroupChangePolicyTestSuite returns fsGroupChangePolicyTestSuite that implements TestSuite interface
func InitCustomFsGroupChangePolicyTestSuite(patterns []storageframework.TestPattern) storageframework.TestSuite {
	return &fsGroupChangePolicyTestSuite{
		tsInfo: storageframework.TestSuiteInfo{
			Name:         "fsgroupchangepolicy",
			TestPatterns: patterns,
			SupportedSizeRange: e2evolume.SizeRange{
				Min: "1Mi",
			},
		},
	}
}

// InitFsGroupChangePolicyTestSuite returns fsGroupChangePolicyTestSuite that implements TestSuite interface
func InitFsGroupChangePolicyTestSuite() storageframework.TestSuite {
	patterns := []storageframework.TestPattern{
		storageframework.DefaultFsDynamicPV,
	}
	return InitCustomFsGroupChangePolicyTestSuite(patterns)
}

func (s *fsGroupChangePolicyTestSuite) GetTestSuiteInfo() storageframework.TestSuiteInfo {
	return s.tsInfo
}

func (s *fsGroupChangePolicyTestSuite) SkipUnsupportedTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) {
	skipVolTypePatterns(pattern, driver, storageframework.NewVolTypeMap(storageframework.CSIInlineVolume, storageframework.GenericEphemeralVolume))
	dInfo := driver.GetDriverInfo()
	if !dInfo.Capabilities[storageframework.CapFsGroup] {
		e2eskipper.Skipf("Driver %q does not support FsGroup - skipping", dInfo.Name)
	}

	if pattern.VolMode == v1.PersistentVolumeBlock {
		e2eskipper.Skipf("Test does not support non-filesystem volume mode - skipping")
	}

	if pattern.VolType != storageframework.DynamicPV {
		e2eskipper.Skipf("Suite %q does not support %v", s.tsInfo.Name, pattern.VolType)
	}

	_, ok := driver.(storageframework.DynamicPVTestDriver)
	if !ok {
		e2eskipper.Skipf("Driver %s doesn't support %v -- skipping", dInfo.Name, pattern.VolType)
	}
}

func (s *fsGroupChangePolicyTestSuite) DefineTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) {
	type local struct {
		config        *storageframework.PerTestConfig
		driverCleanup func()
		driver        storageframework.TestDriver
		resource      *storageframework.VolumeResource
	}
	var l local

	// Beware that it also registers an AfterEach which renders f unusable. Any code using
	// f must run inside an It or Context callback.
	f := framework.NewFrameworkWithCustomTimeouts("fsgroupchangepolicy", storageframework.GetDriverTimeouts(driver))

	init := func() {
		e2eskipper.SkipIfNodeOSDistroIs("windows")
		l = local{}
		l.driver = driver
		l.config, l.driverCleanup = driver.PrepareTest(f)
		testVolumeSizeRange := s.GetTestSuiteInfo().SupportedSizeRange
		l.resource = storageframework.CreateVolumeResource(l.driver, l.config, pattern, testVolumeSizeRange)
	}

	cleanup := func() {
		var errs []error
		if l.resource != nil {
			if err := l.resource.CleanupResource(); err != nil {
				errs = append(errs, err)
			}
			l.resource = nil
		}

		if l.driverCleanup != nil {
			errs = append(errs, storageutils.TryFunc(l.driverCleanup))
			l.driverCleanup = nil
		}

		framework.ExpectNoError(errors.NewAggregate(errs), "while cleanup resource")
	}

	tests := []struct {
		name                              string // Test case name
		podfsGroupChangePolicy            string // 'Always' or 'OnRootMismatch'
		initialPodFsGroup                 int    // FsGroup of the initial pod
		changedRootDirFileOwnership       int    // Change the ownership of the file in the root directory (/mnt/volume1/file1), as part of the initial pod
		changedSubDirFileOwnership        int    // Change the ownership of the file in the sub directory (/mnt/volume1/subdir/file2), as part of the initial pod
		secondPodFsGroup                  int    // FsGroup of the second pod
		finalExpectedRootDirFileOwnership int    // Final expcted ownership of the file in the root directory (/mnt/volume1/file1), as part of the second pod
		finalExpectedSubDirFileOwnership  int    // Final expcted ownership of the file in the sub directory (/mnt/volume1/subdir/file2), as part of the second pod
		// Whether the test can run for drivers that support volumeMountGroup capability.
		// For CSI drivers that support volumeMountGroup:
		// * OnRootMismatch policy is not supported.
		// * It may not be possible to chgrp after mounting a volume.
		supportsVolumeMountGroup bool
	}{
		// Test cases for 'Always' policy
		{
			name:                              "pod created with an initial fsgroup, new pod fsgroup applied to volume contents",
			podfsGroupChangePolicy:            "Always",
			initialPodFsGroup:                 1000,
			secondPodFsGroup:                  2000,
			finalExpectedRootDirFileOwnership: 2000,
			finalExpectedSubDirFileOwnership:  2000,
			supportsVolumeMountGroup:          true,
		},
		{
			name:                              "pod created with an initial fsgroup, volume contents ownership changed via chgrp in first pod, new pod with same fsgroup applied to the volume contents",
			podfsGroupChangePolicy:            "Always",
			initialPodFsGroup:                 1000,
			changedRootDirFileOwnership:       2000,
			changedSubDirFileOwnership:        3000,
			secondPodFsGroup:                  1000,
			finalExpectedRootDirFileOwnership: 1000,
			finalExpectedSubDirFileOwnership:  1000,
		},
		{
			name:                              "pod created with an initial fsgroup, volume contents ownership changed via chgrp in first pod, new pod with different fsgroup applied to the volume contents",
			podfsGroupChangePolicy:            "Always",
			initialPodFsGroup:                 1000,
			changedRootDirFileOwnership:       2000,
			changedSubDirFileOwnership:        3000,
			secondPodFsGroup:                  4000,
			finalExpectedRootDirFileOwnership: 4000,
			finalExpectedSubDirFileOwnership:  4000,
		},
		// Test cases for 'OnRootMismatch' policy
		{
			name:                              "pod created with an initial fsgroup, new pod fsgroup applied to volume contents",
			podfsGroupChangePolicy:            "OnRootMismatch",
			initialPodFsGroup:                 1000,
			secondPodFsGroup:                  2000,
			finalExpectedRootDirFileOwnership: 2000,
			finalExpectedSubDirFileOwnership:  2000,
		},
		{
			name:                              "pod created with an initial fsgroup, volume contents ownership changed via chgrp in first pod, new pod with same fsgroup skips ownership changes to the volume contents",
			podfsGroupChangePolicy:            "OnRootMismatch",
			initialPodFsGroup:                 1000,
			changedRootDirFileOwnership:       2000,
			changedSubDirFileOwnership:        3000,
			secondPodFsGroup:                  1000,
			finalExpectedRootDirFileOwnership: 2000,
			finalExpectedSubDirFileOwnership:  3000,
		},
		{
			name:                              "pod created with an initial fsgroup, volume contents ownership changed via chgrp in first pod, new pod with different fsgroup applied to the volume contents",
			podfsGroupChangePolicy:            "OnRootMismatch",
			initialPodFsGroup:                 1000,
			changedRootDirFileOwnership:       2000,
			changedSubDirFileOwnership:        3000,
			secondPodFsGroup:                  4000,
			finalExpectedRootDirFileOwnership: 4000,
			finalExpectedSubDirFileOwnership:  4000,
		},
	}

	for _, t := range tests {
		test := t
		testCaseName := fmt.Sprintf("(%s)[LinuxOnly], %s", test.podfsGroupChangePolicy, test.name)
		ginkgo.It(testCaseName, func() {
			dInfo := driver.GetDriverInfo()
			policy := v1.PodFSGroupChangePolicy(test.podfsGroupChangePolicy)

			if dInfo.Capabilities[storageframework.CapVolumeMountGroup] &&
				!test.supportsVolumeMountGroup {
				e2eskipper.Skipf("Driver %q supports VolumeMountGroup, which is incompatible with this test - skipping", dInfo.Name)
			}

			init()
			defer cleanup()
			podConfig := e2epod.Config{
				NS:                     f.Namespace.Name,
				NodeSelection:          l.config.ClientNodeSelection,
				PVCs:                   []*v1.PersistentVolumeClaim{l.resource.Pvc},
				FsGroup:                utilpointer.Int64Ptr(int64(test.initialPodFsGroup)),
				PodFSGroupChangePolicy: &policy,
			}
			// Create initial pod and create files in root and sub-directory and verify ownership.
			pod := createPodAndVerifyContentGid(l.config.Framework, &podConfig, true /* createInitialFiles */, "" /* expectedRootDirFileOwnership */, "" /* expectedSubDirFileOwnership */)

			// Change the ownership of files in the initial pod.
			if test.changedRootDirFileOwnership != 0 {
				ginkgo.By(fmt.Sprintf("Changing the root directory file ownership to %s", strconv.Itoa(test.changedRootDirFileOwnership)))
				storageutils.ChangeFilePathGidInPod(f, rootDirFilePath, strconv.Itoa(test.changedRootDirFileOwnership), pod)
			}

			if test.changedSubDirFileOwnership != 0 {
				ginkgo.By(fmt.Sprintf("Changing the sub-directory file ownership to %s", strconv.Itoa(test.changedSubDirFileOwnership)))
				storageutils.ChangeFilePathGidInPod(f, subDirFilePath, strconv.Itoa(test.changedSubDirFileOwnership), pod)
			}

			ginkgo.By(fmt.Sprintf("Deleting Pod %s/%s", pod.Namespace, pod.Name))
			framework.ExpectNoError(e2epod.DeletePodWithWait(f.ClientSet, pod))

			// Create a second pod with existing volume and verify the contents ownership.
			podConfig.FsGroup = utilpointer.Int64Ptr(int64(test.secondPodFsGroup))
			pod = createPodAndVerifyContentGid(l.config.Framework, &podConfig, false /* createInitialFiles */, strconv.Itoa(test.finalExpectedRootDirFileOwnership), strconv.Itoa(test.finalExpectedSubDirFileOwnership))
			ginkgo.By(fmt.Sprintf("Deleting Pod %s/%s", pod.Namespace, pod.Name))
			framework.ExpectNoError(e2epod.DeletePodWithWait(f.ClientSet, pod))
		})
	}
}

func createPodAndVerifyContentGid(f *framework.Framework, podConfig *e2epod.Config, createInitialFiles bool, expectedRootDirFileOwnership, expectedSubDirFileOwnership string) *v1.Pod {
	podFsGroup := strconv.FormatInt(*podConfig.FsGroup, 10)
	ginkgo.By(fmt.Sprintf("Creating Pod in namespace %s with fsgroup %s", podConfig.NS, podFsGroup))
	pod, err := e2epod.CreateSecPodWithNodeSelection(f.ClientSet, podConfig, f.Timeouts.PodStart)
	framework.ExpectNoError(err)
	framework.Logf("Pod %s/%s started successfully", pod.Namespace, pod.Name)

	if createInitialFiles {
		ginkgo.By(fmt.Sprintf("Creating a sub-directory and file, and verifying their ownership is %s", podFsGroup))
		cmd := fmt.Sprintf("touch %s", rootDirFilePath)
		var err error
		_, _, err = e2evolume.PodExec(f, pod, cmd)
		framework.ExpectNoError(err)
		storageutils.VerifyFilePathGidInPod(f, rootDirFilePath, podFsGroup, pod)

		cmd = fmt.Sprintf("mkdir %s", subdir)
		_, _, err = e2evolume.PodExec(f, pod, cmd)
		framework.ExpectNoError(err)
		cmd = fmt.Sprintf("touch %s", subDirFilePath)
		_, _, err = e2evolume.PodExec(f, pod, cmd)
		framework.ExpectNoError(err)
		storageutils.VerifyFilePathGidInPod(f, subDirFilePath, podFsGroup, pod)
		return pod
	}

	// Verify existing contents of the volume
	ginkgo.By(fmt.Sprintf("Verifying the ownership of root directory file is %s", expectedRootDirFileOwnership))
	storageutils.VerifyFilePathGidInPod(f, rootDirFilePath, expectedRootDirFileOwnership, pod)
	ginkgo.By(fmt.Sprintf("Verifying the ownership of sub directory file is %s", expectedSubDirFileOwnership))
	storageutils.VerifyFilePathGidInPod(f, subDirFilePath, expectedSubDirFileOwnership, pod)
	return pod
}
<|endoftext|>


<commit_before>package pod

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"

	"github.com/hyperhq/hypercontainer-utils/hlog"
	"github.com/hyperhq/hyperd/storage"
	dm "github.com/hyperhq/hyperd/storage/devicemapper"
	apitypes "github.com/hyperhq/hyperd/types"
	"github.com/hyperhq/hyperd/utils"
	runv "github.com/hyperhq/runv/api"
)

func GetMountIdByContainer(driver, cid string) (string, error) {
	idPath := path.Join(utils.HYPER_ROOT, fmt.Sprintf("image/%s/layerdb/mounts/%s/mount-id", driver, cid))
	if _, err := os.Stat(idPath); err != nil && os.IsNotExist(err) {
		return "", err
	}

	id, err := ioutil.ReadFile(idPath)
	if err != nil {
		return "", err
	}

	return string(id), nil
}

func ProbeExistingVolume(v *apitypes.UserVolume, sharedDir string) (*runv.VolumeDescription, error) {
	if v == nil || v.Source == "" { //do not create volume in this function, it depends on storage driver.
		return nil, fmt.Errorf("can not generate volume info from %v", v)
	}

	var err error = nil
	vol := &runv.VolumeDescription{
		Name:   v.Name,
		Source: v.Source,
		Format: v.Format,
		Fstype: v.Fstype,
	}

	if v.Option != nil {
		vol.Options = &runv.VolumeOption{
			User:     v.Option.User,
			Monitors: v.Option.Monitors,
			Keyring:  v.Option.Keyring,
		}
	}

	if v.Format == "vfs" {
		vol.Fstype = "dir"
		vol.Source, err = storage.MountVFSVolume(v.Source, sharedDir)
		if err != nil {
			return nil, err
		}
		hlog.Log(DEBUG, "dir %s is bound to %s", v.Source, vol.Source)
	} else if v.Format == "raw" && v.Fstype == "" {
		vol.Fstype, err = dm.ProbeFsType(v.Source)
		if err != nil {
			vol.Fstype = storage.DEFAULT_VOL_FS
			err = nil
		}
	}

	return vol, nil
}

func UmountExistingVolume(fstype, target, sharedDir string) error {
	if fstype == "dir" {
		return storage.UmountVFSVolume(target, sharedDir)
	}
	if !path.IsAbs(target) {
		return nil
	}
	return dm.UnmapVolume(target)
}
<commit_msg>only unmap dm device<commit_after>package pod

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"

	"github.com/hyperhq/hypercontainer-utils/hlog"
	"github.com/hyperhq/hyperd/storage"
	dm "github.com/hyperhq/hyperd/storage/devicemapper"
	apitypes "github.com/hyperhq/hyperd/types"
	"github.com/hyperhq/hyperd/utils"
	runv "github.com/hyperhq/runv/api"
)

func GetMountIdByContainer(driver, cid string) (string, error) {
	idPath := path.Join(utils.HYPER_ROOT, fmt.Sprintf("image/%s/layerdb/mounts/%s/mount-id", driver, cid))
	if _, err := os.Stat(idPath); err != nil && os.IsNotExist(err) {
		return "", err
	}

	id, err := ioutil.ReadFile(idPath)
	if err != nil {
		return "", err
	}

	return string(id), nil
}

func ProbeExistingVolume(v *apitypes.UserVolume, sharedDir string) (*runv.VolumeDescription, error) {
	if v == nil || v.Source == "" { //do not create volume in this function, it depends on storage driver.
		return nil, fmt.Errorf("can not generate volume info from %v", v)
	}

	var err error = nil
	vol := &runv.VolumeDescription{
		Name:   v.Name,
		Source: v.Source,
		Format: v.Format,
		Fstype: v.Fstype,
	}

	if v.Option != nil {
		vol.Options = &runv.VolumeOption{
			User:     v.Option.User,
			Monitors: v.Option.Monitors,
			Keyring:  v.Option.Keyring,
		}
	}

	if v.Format == "vfs" {
		vol.Fstype = "dir"
		vol.Source, err = storage.MountVFSVolume(v.Source, sharedDir)
		if err != nil {
			return nil, err
		}
		hlog.Log(DEBUG, "dir %s is bound to %s", v.Source, vol.Source)
	} else if v.Format == "raw" && v.Fstype == "" {
		vol.Fstype, err = dm.ProbeFsType(v.Source)
		if err != nil {
			vol.Fstype = storage.DEFAULT_VOL_FS
			err = nil
		}
	}

	return vol, nil
}

func UmountExistingVolume(fstype, target, sharedDir string) error {
	if fstype == "dir" {
		return storage.UmountVFSVolume(target, sharedDir)
	}
	if !path.IsAbs(target) {
		return nil
	}

	f, err := os.Stat(target)
	if err == nil && (f.Mode()&os.ModeDevice) != 0 {
		return dm.UnmapVolume(target)
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package conn

import (
	"sync"

	"v.io/v23/context"
	"v.io/v23/flow"
	"v.io/v23/flow/message"
	"v.io/v23/rpc/version"
	"v.io/x/ref/runtime/internal/flow/cipher/aead"
	"v.io/x/ref/runtime/internal/flow/cipher/naclbox"
	"v.io/x/ref/runtime/protocols/lib/framer"
)

// unsafeUnencrypted allows protocol implementors to provide unencrypted
// protocols.  If the underlying connection implements this method, and
// the method returns true and encryption is disabled even if enableEncryption
// is called. This is only used for testing and in particular by the debug
// protocol.
type unsafeUnencrypted interface {
	UnsafeDisableEncryption() bool
}

// newMessagePipe returns a new messagePipe instance that may create its
// own frames on the write path if the supplied MsgReadWriteCloser implements
// framing.T. This offers a significant speedup (half the number of system calls)
// and reduced memory usage and associated allocations.
func newMessagePipe(rw flow.MsgReadWriteCloser) *messagePipe {
	if bypass, ok := rw.(framer.T); ok {
		return &messagePipe{
			rw:          rw,
			framer:      bypass,
			frameOffset: bypass.FrameHeaderSize(),
		}
	}
	return &messagePipe{
		rw: rw,
	}
}

// newMessagePipeUseFramer is like newMessagePipe but will always use
// an external framer, it's included primarily for tests.
func newMessagePipeUseFramer(rw flow.MsgReadWriteCloser) *messagePipe {
	return &messagePipe{
		rw: rw,
	}
}

type sealFunc func(out, data []byte) ([]byte, error)
type openFunc func(out, data []byte) ([]byte, bool)

// messagePipe implements messagePipe for RPC11 version and beyond.
type messagePipe struct {
	rw          flow.MsgReadWriteCloser
	framer      framer.T
	frameOffset int

	// locks are required to serialize access to the cipher operations since
	// the messagePipe may be called by different goroutines when connections
	// are being created or because of the need to send changed blessings
	// asynchronously. Other than these cases there will be no lock
	// contention.
	sealMu sync.Mutex
	seal   sealFunc
	openMu sync.Mutex
	open   openFunc
}

func (p *messagePipe) isEncapsulated() bool {
	_, ok := p.rw.(*flw)
	return ok
}

func (p *messagePipe) disableEncryptionOnEncapsulatedFlow() {
	if f, ok := p.rw.(*flw); ok {
		f.disableEncryption()
	}
}

func (p *messagePipe) Close() error {
	return p.rw.Close()
}

// enableEncryption enables encryption on the pipe (unless the underlying
// transport reader implements UnsafeDisableEncryption and that
// implementatio nreturns true). The encryption used depends on the RPC version
// being used.
func (p *messagePipe) enableEncryption(ctx *context.T, publicKey, secretKey, remotePublicKey *[32]byte, rpcversion version.RPCVersion) ([]byte, error) {
	if uu, ok := p.rw.(unsafeUnencrypted); ok && uu.UnsafeDisableEncryption() {
		return nil, nil
	}
	switch {
	case rpcversion >= version.RPCVersion11 && rpcversion < version.RPCVersion15:
		cipher, err := naclbox.NewCipher(publicKey, secretKey, remotePublicKey)
		if err != nil {
			return nil, err
		}
		p.seal = cipher.Seal
		p.open = cipher.Open
		return cipher.ChannelBinding(), nil
	case rpcversion >= version.RPCVersion15:
		cipher, err := aead.NewCipher(publicKey, secretKey, remotePublicKey)
		if err != nil {
			return nil, err
		}
		p.seal = cipher.Seal
		p.open = cipher.Open
		return cipher.ChannelBinding(), nil
	}
	return nil, ErrRPCVersionMismatch.Errorf(ctx, "conn.message_pipe: %v is not supported", rpcversion)
}

func usedOurBuffer(x, y []byte) bool {
	return &x[0:cap(x)][cap(x)-1] == &y[0:cap(y)][cap(y)-1]
}

func (p *messagePipe) writeCiphertext(ctx *context.T, m message.Message, plaintextBuf, ciphertextBuf []byte) (wire, framed []byte, err error) {
	if p.seal == nil {
		wire, err = message.Append(ctx, m, plaintextBuf[p.frameOffset:p.frameOffset])
		framed = plaintextBuf
		return
	}
	plaintext, err := message.Append(ctx, m, plaintextBuf[:0])
	if err != nil {
		return
	}
	p.sealMu.Lock()
	wire, err = p.seal(ciphertextBuf[p.frameOffset:p.frameOffset], plaintext)
	p.sealMu.Unlock()
	framed = ciphertextBuf
	return
}

func (p *messagePipe) writeMsg(ctx *context.T, m message.Message) error {
	plaintextBuf := messagePipePool.Get().(*[]byte)
	defer messagePipePool.Put(plaintextBuf)

	ciphertextBuf := messagePipePool.Get().(*[]byte)
	defer messagePipePool.Put(ciphertextBuf)

	plaintextPayload, ok := message.PlaintextPayload(m)
	if ok && len(plaintextPayload) == 0 {
		message.ClearDisableEncryptionFlag(m)
	}

	wire, framedWire, err := p.writeCiphertext(ctx, m, *plaintextBuf, *ciphertextBuf)
	if err != nil {
		return err
	}

	if p.frameOffset > 0 && usedOurBuffer(framedWire, wire) {
		// Write the frame size directly into the buffer we allocated and then
		// write out that buffer in a single write operation.
		if err := p.framer.PutSize(framedWire[:p.frameOffset], len(wire)); err != nil {
			return err
		}
		if _, err = p.framer.Write(framedWire[:len(wire)+p.frameOffset]); err != nil {
			return err
		}
	} else {
		// NOTE that in the case where p.frameOffset > 0 but the returned buffer
		// differs from the one passed in, p.frameOffset bytes are wasted in the
		// buffers used here.
		if _, err = p.rw.WriteMsg(wire); err != nil {
			return err
		}
	}

	// Handle plaintext payloads which are not serialized by message.Append
	// above and are instead written separately in the clear.
	if plaintextPayload != nil {
		if _, err = p.rw.WriteMsg(plaintextPayload...); err != nil {
			return err
		}
	}
	if ctx.V(2) {
		ctx.Infof("Wrote low-level message: %T: %v", m, m)
	}
	return nil
}

func (p *messagePipe) readClearText(ctx *context.T, plaintextBuf []byte) ([]byte, error) {
	if p.open == nil {
		return p.rw.ReadMsg2(plaintextBuf)
	}
	ciphertextBuf := messagePipePool.Get().(*[]byte)
	defer messagePipePool.Put(ciphertextBuf)
	ciphertext, err := p.rw.ReadMsg2(*ciphertextBuf)
	if err != nil {
		return nil, err
	}
	p.openMu.Lock()
	plaintext, ok := p.open(plaintextBuf[:0], ciphertext)
	p.openMu.Unlock()
	if !ok {
		return nil, message.NewErrInvalidMsg(ctx, 0, uint64(len(ciphertext)), 0, nil)
	}
	return plaintext, nil
}

func (p *messagePipe) readMsg(ctx *context.T, plaintextBuf []byte) (message.Message, error) {
	plaintext, err := p.readClearText(ctx, plaintextBuf)
	if err != nil {
		return nil, err
	}
	return p.readAnyMessage(ctx, plaintext)
}

func (p *messagePipe) readAnyMessage(ctx *context.T, plaintext []byte) (message.Message, error) {
	m, err := message.Read(ctx, plaintext)
	if err != nil {
		return nil, err
	}
	if message.ExpectsPlaintextPayload(m) {
		payload, err := p.rw.ReadMsg2(nil)
		if err != nil {
			return nil, err
		}
		// nocopy is set to true since the buffer was newly allocated here.
		message.SetPlaintextPayload(m, payload, true)
	}
	if ctx.V(2) {
		ctx.Infof("Read low-level message: %T: %v", m, m)
	}
	return m, err
}

func (p *messagePipe) readDataMsg(ctx *context.T, plaintextBuf []byte, m *message.Data) (message.Message, error) {
	plaintext, err := p.readClearText(ctx, plaintextBuf)
	if err != nil {
		return nil, err
	}
	if ok, err := message.ReadData(ctx, plaintext, m); !ok {
		if err != nil {
			return nil, err
		}
		return p.readAnyMessage(ctx, plaintext)
	}
	if m.Flags&message.DisableEncryptionFlag != 0 {
		payload, err := p.rw.ReadMsg2(nil)
		if err != nil {
			return nil, err
		}
		message.SetPlaintextDataPayload(m, payload, true)
	}
	if ctx.V(2) {
		ctx.Infof("Read low-level message: %T: %v", m, m)
	}
	return nil, nil
}
<commit_msg>x/ref/runtime/internal/flow/conn: extend/simplify locking (#334)<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package conn

import (
	"sync"

	"v.io/v23/context"
	"v.io/v23/flow"
	"v.io/v23/flow/message"
	"v.io/v23/rpc/version"
	"v.io/x/ref/runtime/internal/flow/cipher/aead"
	"v.io/x/ref/runtime/internal/flow/cipher/naclbox"
	"v.io/x/ref/runtime/protocols/lib/framer"
)

// unsafeUnencrypted allows protocol implementors to provide unencrypted
// protocols.  If the underlying connection implements this method, and
// the method returns true and encryption is disabled even if enableEncryption
// is called. This is only used for testing and in particular by the debug
// protocol.
type unsafeUnencrypted interface {
	UnsafeDisableEncryption() bool
}

// newMessagePipe returns a new messagePipe instance that may create its
// own frames on the write path if the supplied MsgReadWriteCloser implements
// framing.T. This offers a significant speedup (half the number of system calls)
// and reduced memory usage and associated allocations.
func newMessagePipe(rw flow.MsgReadWriteCloser) *messagePipe {
	if bypass, ok := rw.(framer.T); ok {
		return &messagePipe{
			rw:          rw,
			framer:      bypass,
			frameOffset: bypass.FrameHeaderSize(),
		}
	}
	return &messagePipe{
		rw: rw,
	}
}

// newMessagePipeUseFramer is like newMessagePipe but will always use
// an external framer, it's included primarily for tests.
func newMessagePipeUseFramer(rw flow.MsgReadWriteCloser) *messagePipe {
	return &messagePipe{
		rw: rw,
	}
}

type sealFunc func(out, data []byte) ([]byte, error)
type openFunc func(out, data []byte) ([]byte, bool)

// messagePipe implements messagePipe for RPC11 version and beyond.
type messagePipe struct {
	rw          flow.MsgReadWriteCloser
	framer      framer.T
	frameOffset int

	seal sealFunc
	open openFunc

	// locks are required to serialize access to the read/write operations since
	// the messagePipe may be called by different goroutines when connections
	// are being created or because of the need to send changed blessings
	// asynchronously. Other than these cases there will be no lock
	// contention.
	readMu, writeMu sync.Mutex
}

func (p *messagePipe) isEncapsulated() bool {
	_, ok := p.rw.(*flw)
	return ok
}

func (p *messagePipe) disableEncryptionOnEncapsulatedFlow() {
	if f, ok := p.rw.(*flw); ok {
		f.disableEncryption()
	}
}

func (p *messagePipe) Close() error {
	return p.rw.Close()
}

// enableEncryption enables encryption on the pipe (unless the underlying
// transport reader implements UnsafeDisableEncryption and that
// implementatio nreturns true). The encryption used depends on the RPC version
// being used.
func (p *messagePipe) enableEncryption(ctx *context.T, publicKey, secretKey, remotePublicKey *[32]byte, rpcversion version.RPCVersion) ([]byte, error) {
	if uu, ok := p.rw.(unsafeUnencrypted); ok && uu.UnsafeDisableEncryption() {
		return nil, nil
	}
	switch {
	case rpcversion >= version.RPCVersion11 && rpcversion < version.RPCVersion15:
		cipher, err := naclbox.NewCipher(publicKey, secretKey, remotePublicKey)
		if err != nil {
			return nil, err
		}
		p.seal = cipher.Seal
		p.open = cipher.Open
		return cipher.ChannelBinding(), nil
	case rpcversion >= version.RPCVersion15:
		cipher, err := aead.NewCipher(publicKey, secretKey, remotePublicKey)
		if err != nil {
			return nil, err
		}
		p.seal = cipher.Seal
		p.open = cipher.Open
		return cipher.ChannelBinding(), nil
	}
	return nil, ErrRPCVersionMismatch.Errorf(ctx, "conn.message_pipe: %v is not supported", rpcversion)
}

func usedOurBuffer(x, y []byte) bool {
	return &x[0:cap(x)][cap(x)-1] == &y[0:cap(y)][cap(y)-1]
}

func (p *messagePipe) writeCiphertext(ctx *context.T, m message.Message, plaintextBuf, ciphertextBuf []byte) (wire, framed []byte, err error) {
	if p.seal == nil {
		wire, err = message.Append(ctx, m, plaintextBuf[p.frameOffset:p.frameOffset])
		framed = plaintextBuf
		return
	}
	plaintext, err := message.Append(ctx, m, plaintextBuf[:0])
	if err != nil {
		return
	}
	wire, err = p.seal(ciphertextBuf[p.frameOffset:p.frameOffset], plaintext)
	framed = ciphertextBuf
	return
}

func (p *messagePipe) writeMsg(ctx *context.T, m message.Message) error {
	plaintextBuf := messagePipePool.Get().(*[]byte)
	defer messagePipePool.Put(plaintextBuf)

	ciphertextBuf := messagePipePool.Get().(*[]byte)
	defer messagePipePool.Put(ciphertextBuf)

	plaintextPayload, ok := message.PlaintextPayload(m)
	if ok && len(plaintextPayload) == 0 {
		message.ClearDisableEncryptionFlag(m)
	}

	p.writeMu.Lock()
	defer p.writeMu.Unlock()
	wire, framedWire, err := p.writeCiphertext(ctx, m, *plaintextBuf, *ciphertextBuf)
	if err != nil {
		return err
	}

	if p.frameOffset > 0 && usedOurBuffer(framedWire, wire) {
		// Write the frame size directly into the buffer we allocated and then
		// write out that buffer in a single write operation.
		if err := p.framer.PutSize(framedWire[:p.frameOffset], len(wire)); err != nil {
			return err
		}
		if _, err = p.framer.Write(framedWire[:len(wire)+p.frameOffset]); err != nil {
			return err
		}
	} else {
		// NOTE that in the case where p.frameOffset > 0 but the returned buffer
		// differs from the one passed in, p.frameOffset bytes are wasted in the
		// buffers used here.
		if _, err = p.rw.WriteMsg(wire); err != nil {
			return err
		}
	}

	// Handle plaintext payloads which are not serialized by message.Append
	// above and are instead written separately in the clear.
	if plaintextPayload != nil {
		if _, err = p.rw.WriteMsg(plaintextPayload...); err != nil {
			return err
		}
	}
	if ctx.V(2) {
		ctx.Infof("Wrote low-level message: %T: %v", m, m)
	}
	return nil
}

func (p *messagePipe) readClearText(ctx *context.T, plaintextBuf []byte) ([]byte, error) {
	if p.open == nil {
		return p.rw.ReadMsg2(plaintextBuf)
	}
	ciphertextBuf := messagePipePool.Get().(*[]byte)
	defer messagePipePool.Put(ciphertextBuf)
	ciphertext, err := p.rw.ReadMsg2(*ciphertextBuf)
	if err != nil {
		return nil, err
	}
	plaintext, ok := p.open(plaintextBuf[:0], ciphertext)
	if !ok {
		return nil, message.NewErrInvalidMsg(ctx, 0, uint64(len(ciphertext)), 0, nil)
	}
	return plaintext, nil
}

func (p *messagePipe) readMsg(ctx *context.T, plaintextBuf []byte) (message.Message, error) {
	p.readMu.Lock()
	plaintext, err := p.readClearText(ctx, plaintextBuf)
	if err != nil {
		p.readMu.Unlock()
		return nil, err
	}
	p.readMu.Unlock()
	return p.readAnyMessage(ctx, plaintext)
}

func (p *messagePipe) readAnyMessage(ctx *context.T, plaintext []byte) (message.Message, error) {
	m, err := message.Read(ctx, plaintext)
	if err != nil {
		return nil, err
	}
	if message.ExpectsPlaintextPayload(m) {
		payload, err := p.rw.ReadMsg2(nil)
		if err != nil {
			return nil, err
		}
		// nocopy is set to true since the buffer was newly allocated here.
		message.SetPlaintextPayload(m, payload, true)
	}
	if ctx.V(2) {
		ctx.Infof("Read low-level message: %T: %v", m, m)
	}
	return m, err
}

func (p *messagePipe) readDataMsg(ctx *context.T, plaintextBuf []byte, m *message.Data) (message.Message, error) {
	p.readMu.Lock()
	plaintext, err := p.readClearText(ctx, plaintextBuf)
	if err != nil {
		p.readMu.Unlock()
		return nil, err
	}
	p.readMu.Unlock()
	if ok, err := message.ReadData(ctx, plaintext, m); !ok {
		if err != nil {
			return nil, err
		}
		return p.readAnyMessage(ctx, plaintext)
	}
	if m.Flags&message.DisableEncryptionFlag != 0 {
		payload, err := p.rw.ReadMsg2(nil)
		if err != nil {
			return nil, err
		}
		message.SetPlaintextDataPayload(m, payload, true)
	}
	if ctx.V(2) {
		ctx.Infof("Read low-level message: %T: %v", m, m)
	}
	return nil, nil
}
<|endoftext|>


<commit_before>package stream

import "io"
import "math"

func ChunkLimitReader(reader io.Reader, chunkSize, totalSize int64) ChunkedLimitedReader {
    numChunks := int64(math.Ceil(float64(totalSize) / float64(chunkSize)))
    return ChunkedLimitedReader {
        d: &clrData {
            reader:     reader,
            remaining:  totalSize,
            chunkSize:  chunkSize,
            chunkRem:   chunkSize,
            numChunks:  numChunks,
            doneChunks: 0,
        },
    }
}

// will read *past* the end of the total size to fill in the remainder of a chunk
// effectively acts as a chunk iterator over the input stream
type ChunkedLimitedReader struct {
    d *clrData
}

type clrData struct {
    reader     io.Reader // underlying reader
    remaining  int64     // bytes remaining in total
    chunkSize  int64     // chunk size
    chunkRem   int64     // bytes remaining in chunk
    numChunks  int64     // total number of chunks to allow
    doneChunks int64     // number of chunks completed
}

func (c ChunkedLimitedReader) Read(p []byte) (n int, err error) {
    // If we've already read all our chunks and the remainders are <= 0, we're done
    if c.d.doneChunks >= c.d.numChunks || (c.d.remaining <= 0 && c.d.chunkRem <= 0) {
        return 0, io.EOF
    }

    // Data is done, returning only buffer bytes now
    if c.d.remaining <= 0 {
        if int64(len(p)) > c.d.chunkRem {
            p = p[0:c.d.chunkRem]
        }

        for i := range p {
            p[i] = 0
        }

        c.d.chunkRem -= int64(len(p))
        return len(p), nil
    }

    // Data is not yet done, but chunk is
    if c.d.chunkRem <= 0 {
        return 0, io.EOF
    }

    // Data and chunk not yet done, need to read from outside reader
    if int64(len(p)) > c.d.remaining || int64(len(p)) > c.d.chunkRem {
        rem := int64(math.Min(float64(c.d.remaining), float64(c.d.chunkRem)))
        p = p[0:rem]
    }

    n, err = c.d.reader.Read(p)
    c.d.remaining -= int64(n)
    c.d.chunkRem -= int64(n)

    return
}

func (c *ChunkedLimitedReader) NextChunk() {
    if (c.d.doneChunks < c.d.numChunks) {
        c.d.doneChunks++
        c.d.chunkRem = c.d.chunkSize
    }
}

func (c *ChunkedLimitedReader) More() bool {
    return c.d.doneChunks < c.d.numChunks
}
<commit_msg>Adding some important commenting about the ChunkedLimitedReader<commit_after>package stream

import "io"
import "math"

func ChunkLimitReader(reader io.Reader, chunkSize, totalSize int64) ChunkedLimitedReader {
    numChunks := int64(math.Ceil(float64(totalSize) / float64(chunkSize)))
    return ChunkedLimitedReader {
        d: &clrData {
            reader:     reader,
            remaining:  totalSize,
            chunkSize:  chunkSize,
            chunkRem:   chunkSize,
            numChunks:  numChunks,
            doneChunks: 0,
        },
    }
}

// This reader is ***NOT THREAD SAFE***
// It will read *past* the end of the total size to fill in the remainder of a chunk
// effectively acts as a chunk iterator over the input stream
type ChunkedLimitedReader struct {
    d *clrData
}

// Because the Read method is value-only, we need to keep our state in another struct
// and maintain a pointer to that struct in the main reader struct.
type clrData struct {
    reader     io.Reader // underlying reader
    remaining  int64     // bytes remaining in total
    chunkSize  int64     // chunk size
    chunkRem   int64     // bytes remaining in chunk
    numChunks  int64     // total number of chunks to allow
    doneChunks int64     // number of chunks completed
}

// io.Reader's interface implements this as a value method, not a pointer method.
func (c ChunkedLimitedReader) Read(p []byte) (n int, err error) {
    // If we've already read all our chunks and the remainders are <= 0, we're done
    if c.d.doneChunks >= c.d.numChunks || (c.d.remaining <= 0 && c.d.chunkRem <= 0) {
        return 0, io.EOF
    }

    // Data is done, returning only buffer bytes now
    if c.d.remaining <= 0 {
        if int64(len(p)) > c.d.chunkRem {
            p = p[0:c.d.chunkRem]
        }

        for i := range p {
            p[i] = 0
        }

        c.d.chunkRem -= int64(len(p))
        return len(p), nil
    }

    // Data is not yet done, but chunk is
    if c.d.chunkRem <= 0 {
        return 0, io.EOF
    }

    // Data and chunk not yet done, need to read from outside reader
    if int64(len(p)) > c.d.remaining || int64(len(p)) > c.d.chunkRem {
        rem := int64(math.Min(float64(c.d.remaining), float64(c.d.chunkRem)))
        p = p[0:rem]
    }

    n, err = c.d.reader.Read(p)
    c.d.remaining -= int64(n)
    c.d.chunkRem -= int64(n)

    return
}

func (c *ChunkedLimitedReader) NextChunk() {
    if (c.d.doneChunks < c.d.numChunks) {
        c.d.doneChunks++
        c.d.chunkRem = c.d.chunkSize
    }
}

func (c *ChunkedLimitedReader) More() bool {
    return c.d.doneChunks < c.d.numChunks
}
<|endoftext|>


<commit_before>package helpers

import (
  "io/ioutil"
)

func CreateTempDockerComposeFile(yaml string) string {
  fh, err := ioutil.TempFile("", "lc_docker_compose_template")
  if err != nil {
    panic("could not create temporary yaml file")
  }
  defer fh.Close()
  _, err = fh.WriteString(yaml)
  if err != nil {
    panic("could not write to temporary yaml file")
  }
  return fh.Name()
}
<commit_msg>work around https://github.com/docker/compose/issues/2289<commit_after>package helpers

import (
  "io/ioutil"
  "os"
)

func CreateTempDockerComposeFile(yaml string) string {
  cwd, _ := os.Getwd()
  fh, err := ioutil.TempFile(cwd, "lc_docker_compose_template")
  if err != nil {
    panic("could not create temporary yaml file")
  }
  defer fh.Close()
  _, err = fh.WriteString(yaml)
  if err != nil {
    panic("could not write to temporary yaml file")
  }
  return fh.Name()
}
<|endoftext|>


<commit_before>package server

import (
	"errors"
	"net"

	"github.com/xgfone/go-tools/pool"
)

type UHandle interface {
	// Handle the request from the client, and return the handled result.
	//
	// buf is the data received from the client.
	// addr is the address of the client.
	Handle(buf []byte, addr *net.UDPAddr) []byte
}

func UDPWithError(conn *net.UDPConn, p *pool.BufPool, handle interface{}, buf []byte, addr *net.UDPAddr) {
	yes := true
	defer func() {
		if p != nil {
			p.Put(buf)
		}

		if err := recover(); err != nil {
			_logger.Printf("[Error] Get a error: %v", err)
			if !yes {
				panic(err)
			}
		}
	}()

	var result []byte
	if handler, ok := handle.(UHandle); ok {
		result = handler.Handle(buf, addr)
	} else if handler, ok := handle.(func([]byte, *net.UDPAddr) []byte); ok {
		result = handler(buf, addr)
	} else {
		yes = false
		panic("Don't support the handler")
	}

	// If returning nil, don't send the response to the client.
	if result == nil {
		return
	}

	if _, err := conn.WriteToUDP(result, addr); err != nil {
		_logger.Printf("[Error] Failed to send the data to %s: %v", addr, err)
	}
}

// Start a UDP server and never return. Return an error if returns.
// But if wrap exists and returns true, return nil. Or continue to execute.
//
// network MUST be "udp", "udp4", "udp6".
// addr is like "host:port", such as "127.0.0.1:8000", and host or port
// may be omitted.
// size is the size of the buffer.
func UDPServerForever(network, addr string, size int, handle interface{}, wrap func(*net.UDPConn) bool) error {
	var conn *net.UDPConn
	if _addr, err := net.ResolveUDPAddr(network, addr); err != nil {
		return err
	} else {
		if conn, err = net.ListenUDP(network, _addr); err != nil {
			return err
		}
	}

	defer conn.Close()

	if size < 1 || size > 65536 {
		return errors.New("The size of the buffer is limited between 1 and 65536.")
	}

	if handle == nil && wrap == nil {
		return errors.New("handle and wrap neither be nil.")
	}

	_logger.Printf("[Debug] Listen on %v", addr)

	if wrap != nil {
		if wrap(conn) {
			return nil
		}
	}

	if handle == nil {
		return nil
	}

	_pool := pool.NewBufPool(size)

	for {
		buf := _pool.Get()
		num, caddr, err := conn.ReadFromUDP(buf)
		if err != nil {
			_logger.Printf("[Error] Failed to read the UDP data: %v", err)
		} else {
			go UDPWithError(conn, _pool, handle, buf[:num], caddr)
		}
	}

	// Never execute forever.
	return nil
}
<commit_msg>Update the comment<commit_after>package server

import (
	"errors"
	"net"

	"github.com/xgfone/go-tools/pool"
)

type UHandle interface {
	// Handle the request from the client, and return the handled result.
	//
	// buf is the data received from the client.
	// addr is the address of the client.
	Handle(buf []byte, addr *net.UDPAddr) []byte
}

func UDPWithError(conn *net.UDPConn, p *pool.BufPool, handle interface{}, buf []byte, addr *net.UDPAddr) {
	yes := true
	defer func() {
		if p != nil {
			p.Put(buf)
		}

		if err := recover(); err != nil {
			_logger.Printf("[Error] Get a error: %v", err)
			if !yes {
				panic(err)
			}
		}
	}()

	var result []byte
	if handler, ok := handle.(UHandle); ok {
		result = handler.Handle(buf, addr)
	} else if handler, ok := handle.(func([]byte, *net.UDPAddr) []byte); ok {
		result = handler(buf, addr)
	} else {
		yes = false
		panic("Don't support the handler")
	}

	// If returning nil, don't send the response to the client.
	if result == nil {
		return
	}

	if _, err := conn.WriteToUDP(result, addr); err != nil {
		_logger.Printf("[Error] Failed to send the data to %s: %v", addr, err)
	}
}

// Start a UDP server and never return. Return an error if returns.
// But there is one exception: if wrap exists and returns true, it returns nil.
// Or continue to execute and never return.
//
// network MUST be "udp", "udp4", "udp6".
// addr is like "host:port", such as "127.0.0.1:8000", and host or port
// may be omitted.
// size is the size of the buffer.
func UDPServerForever(network, addr string, size int, handle interface{}, wrap func(*net.UDPConn) bool) error {
	var conn *net.UDPConn
	if _addr, err := net.ResolveUDPAddr(network, addr); err != nil {
		return err
	} else {
		if conn, err = net.ListenUDP(network, _addr); err != nil {
			return err
		}
	}

	defer conn.Close()

	if size < 1 || size > 65536 {
		return errors.New("The size of the buffer is limited between 1 and 65536.")
	}

	if handle == nil && wrap == nil {
		return errors.New("handle and wrap neither be nil.")
	}

	_logger.Printf("[Debug] Listen on %v", addr)

	if wrap != nil {
		if wrap(conn) {
			return nil
		}
	}

	if handle == nil {
		return nil
	}

	_pool := pool.NewBufPool(size)

	for {
		buf := _pool.Get()
		num, caddr, err := conn.ReadFromUDP(buf)
		if err != nil {
			_logger.Printf("[Error] Failed to read the UDP data: %v", err)
		} else {
			go UDPWithError(conn, _pool, handle, buf[:num], caddr)
		}
	}

	// Never execute forever.
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package zip

import (
	"bytes"
	"encoding/binary"
	"io"
	"io/ioutil"
	"os"
	"testing"
	"time"
)

type ZipTest struct {
	Name    string
	Comment string
	File    []ZipTestFile
	Error   error // the error that Opening this file should return
}

type ZipTestFile struct {
	Name    string
	Content []byte // if blank, will attempt to compare against File
	File    string // name of file to compare to (relative to testdata/)
	Mtime   string // modified time in format "mm-dd-yy hh:mm:ss"
	Mode    os.FileMode
}

// Caution: The Mtime values found for the test files should correspond to
//          the values listed with unzip -l <zipfile>. However, the values
//          listed by unzip appear to be off by some hours. When creating
//          fresh test files and testing them, this issue is not present.
//          The test files were created in Sydney, so there might be a time
//          zone issue. The time zone information does have to be encoded
//          somewhere, because otherwise unzip -l could not provide a different
//          time from what the archive/zip package provides, but there appears
//          to be no documentation about this.

var tests = []ZipTest{
	{
		Name:    "test.zip",
		Comment: "This is a zipfile comment.",
		File: []ZipTestFile{
			{
				Name:    "test.txt",
				Content: []byte("This is a test text file.\n"),
				Mtime:   "09-05-10 12:12:02",
				Mode:    0644,
			},
			{
				Name:  "gophercolor16x16.png",
				File:  "gophercolor16x16.png",
				Mtime: "09-05-10 15:52:58",
				Mode:  0644,
			},
		},
	},
	{
		Name: "r.zip",
		File: []ZipTestFile{
			{
				Name:  "r/r.zip",
				File:  "r.zip",
				Mtime: "03-04-10 00:24:16",
				Mode:  0666,
			},
		},
	},
	{Name: "readme.zip"},
	{Name: "readme.notzip", Error: FormatError},
	{
		Name: "dd.zip",
		File: []ZipTestFile{
			{
				Name:    "filename",
				Content: []byte("This is a test textfile.\n"),
				Mtime:   "02-02-11 13:06:20",
				Mode:    0666,
			},
		},
	},
	{
		// created in windows XP file manager.
		Name: "winxp.zip",
		File: crossPlatform,
	},
	{
		// created by Zip 3.0 under Linux
		Name: "unix.zip",
		File: crossPlatform,
	},
}

var crossPlatform = []ZipTestFile{
	{
		Name:    "hello",
		Content: []byte("world \r\n"),
		Mode:    0666,
	},
	{
		Name:    "dir/bar",
		Content: []byte("foo \r\n"),
		Mode:    0666,
	},
	{
		Name:    "dir/empty/",
		Content: []byte{},
		Mode:    os.ModeDir | 0777,
	},
	{
		Name:    "readonly",
		Content: []byte("important \r\n"),
		Mode:    0444,
	},
}

func TestReader(t *testing.T) {
	for _, zt := range tests {
		readTestZip(t, zt)
	}
}

func readTestZip(t *testing.T, zt ZipTest) {
	z, err := OpenReader("testdata/" + zt.Name)
	if err != zt.Error {
		t.Errorf("error=%v, want %v", err, zt.Error)
		return
	}

	// bail if file is not zip
	if err == FormatError {
		return
	}
	defer func() {
		if err := z.Close(); err != nil {
			t.Errorf("error %q when closing zip file", err)
		}
	}()

	// bail here if no Files expected to be tested
	// (there may actually be files in the zip, but we don't care)
	if zt.File == nil {
		return
	}

	if z.Comment != zt.Comment {
		t.Errorf("%s: comment=%q, want %q", zt.Name, z.Comment, zt.Comment)
	}
	if len(z.File) != len(zt.File) {
		t.Errorf("%s: file count=%d, want %d", zt.Name, len(z.File), len(zt.File))
	}

	// test read of each file
	for i, ft := range zt.File {
		readTestFile(t, ft, z.File[i])
	}

	// test simultaneous reads
	n := 0
	done := make(chan bool)
	for i := 0; i < 5; i++ {
		for j, ft := range zt.File {
			go func() {
				readTestFile(t, ft, z.File[j])
				done <- true
			}()
			n++
		}
	}
	for ; n > 0; n-- {
		<-done
	}

	// test invalid checksum
	if !z.File[0].hasDataDescriptor() { // skip test when crc32 in dd
		z.File[0].CRC32++ // invalidate
		r, err := z.File[0].Open()
		if err != nil {
			t.Error(err)
			return
		}
		var b bytes.Buffer
		_, err = io.Copy(&b, r)
		if err != ChecksumError {
			t.Errorf("%s: copy error=%v, want %v", z.File[0].Name, err, ChecksumError)
		}
	}
}

func readTestFile(t *testing.T, ft ZipTestFile, f *File) {
	if f.Name != ft.Name {
		t.Errorf("name=%q, want %q", f.Name, ft.Name)
	}

	if ft.Mtime != "" {
		mtime, err := time.Parse("01-02-06 15:04:05", ft.Mtime)
		if err != nil {
			t.Error(err)
			return
		}
		if ft := f.ModTime(); !ft.Equal(mtime) {
			t.Errorf("%s: mtime=%s, want %s", f.Name, ft, mtime)
		}
	}

	testFileMode(t, f, ft.Mode)

	size0 := f.UncompressedSize

	var b bytes.Buffer
	r, err := f.Open()
	if err != nil {
		t.Error(err)
		return
	}

	if size1 := f.UncompressedSize; size0 != size1 {
		t.Errorf("file %q changed f.UncompressedSize from %d to %d", f.Name, size0, size1)
	}

	_, err = io.Copy(&b, r)
	if err != nil {
		t.Error(err)
		return
	}
	r.Close()

	var c []byte
	if ft.Content != nil {
		c = ft.Content
	} else if c, err = ioutil.ReadFile("testdata/" + ft.File); err != nil {
		t.Error(err)
		return
	}

	if b.Len() != len(c) {
		t.Errorf("%s: len=%d, want %d", f.Name, b.Len(), len(c))
		return
	}

	for i, b := range b.Bytes() {
		if b != c[i] {
			t.Errorf("%s: content[%d]=%q want %q", f.Name, i, b, c[i])
			return
		}
	}
}

func testFileMode(t *testing.T, f *File, want os.FileMode) {
	mode, err := f.Mode()
	if want == 0 {
		if err == nil {
			t.Errorf("%s mode: got %v, want none", f.Name, mode)
		}
	} else if err != nil {
		t.Errorf("%s mode: %s", f.Name, err)
	} else if mode != want {
		t.Errorf("%s mode: want %v, got %v", f.Name, want, mode)
	}
}

func TestInvalidFiles(t *testing.T) {
	const size = 1024 * 70 // 70kb
	b := make([]byte, size)

	// zeroes
	_, err := NewReader(sliceReaderAt(b), size)
	if err != FormatError {
		t.Errorf("zeroes: error=%v, want %v", err, FormatError)
	}

	// repeated directoryEndSignatures
	sig := make([]byte, 4)
	binary.LittleEndian.PutUint32(sig, directoryEndSignature)
	for i := 0; i < size-4; i += 4 {
		copy(b[i:i+4], sig)
	}
	_, err = NewReader(sliceReaderAt(b), size)
	if err != FormatError {
		t.Errorf("sigs: error=%v, want %v", err, FormatError)
	}
}

type sliceReaderAt []byte

func (r sliceReaderAt) ReadAt(b []byte, off int64) (int, error) {
	copy(b, r[int(off):int(off)+len(b)])
	return len(b), nil
}
<commit_msg>zip: fix data race in test<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package zip

import (
	"bytes"
	"encoding/binary"
	"io"
	"io/ioutil"
	"os"
	"testing"
	"time"
)

type ZipTest struct {
	Name    string
	Comment string
	File    []ZipTestFile
	Error   error // the error that Opening this file should return
}

type ZipTestFile struct {
	Name    string
	Content []byte // if blank, will attempt to compare against File
	File    string // name of file to compare to (relative to testdata/)
	Mtime   string // modified time in format "mm-dd-yy hh:mm:ss"
	Mode    os.FileMode
}

// Caution: The Mtime values found for the test files should correspond to
//          the values listed with unzip -l <zipfile>. However, the values
//          listed by unzip appear to be off by some hours. When creating
//          fresh test files and testing them, this issue is not present.
//          The test files were created in Sydney, so there might be a time
//          zone issue. The time zone information does have to be encoded
//          somewhere, because otherwise unzip -l could not provide a different
//          time from what the archive/zip package provides, but there appears
//          to be no documentation about this.

var tests = []ZipTest{
	{
		Name:    "test.zip",
		Comment: "This is a zipfile comment.",
		File: []ZipTestFile{
			{
				Name:    "test.txt",
				Content: []byte("This is a test text file.\n"),
				Mtime:   "09-05-10 12:12:02",
				Mode:    0644,
			},
			{
				Name:  "gophercolor16x16.png",
				File:  "gophercolor16x16.png",
				Mtime: "09-05-10 15:52:58",
				Mode:  0644,
			},
		},
	},
	{
		Name: "r.zip",
		File: []ZipTestFile{
			{
				Name:  "r/r.zip",
				File:  "r.zip",
				Mtime: "03-04-10 00:24:16",
				Mode:  0666,
			},
		},
	},
	{Name: "readme.zip"},
	{Name: "readme.notzip", Error: FormatError},
	{
		Name: "dd.zip",
		File: []ZipTestFile{
			{
				Name:    "filename",
				Content: []byte("This is a test textfile.\n"),
				Mtime:   "02-02-11 13:06:20",
				Mode:    0666,
			},
		},
	},
	{
		// created in windows XP file manager.
		Name: "winxp.zip",
		File: crossPlatform,
	},
	{
		// created by Zip 3.0 under Linux
		Name: "unix.zip",
		File: crossPlatform,
	},
}

var crossPlatform = []ZipTestFile{
	{
		Name:    "hello",
		Content: []byte("world \r\n"),
		Mode:    0666,
	},
	{
		Name:    "dir/bar",
		Content: []byte("foo \r\n"),
		Mode:    0666,
	},
	{
		Name:    "dir/empty/",
		Content: []byte{},
		Mode:    os.ModeDir | 0777,
	},
	{
		Name:    "readonly",
		Content: []byte("important \r\n"),
		Mode:    0444,
	},
}

func TestReader(t *testing.T) {
	for _, zt := range tests {
		readTestZip(t, zt)
	}
}

func readTestZip(t *testing.T, zt ZipTest) {
	z, err := OpenReader("testdata/" + zt.Name)
	if err != zt.Error {
		t.Errorf("error=%v, want %v", err, zt.Error)
		return
	}

	// bail if file is not zip
	if err == FormatError {
		return
	}
	defer func() {
		if err := z.Close(); err != nil {
			t.Errorf("error %q when closing zip file", err)
		}
	}()

	// bail here if no Files expected to be tested
	// (there may actually be files in the zip, but we don't care)
	if zt.File == nil {
		return
	}

	if z.Comment != zt.Comment {
		t.Errorf("%s: comment=%q, want %q", zt.Name, z.Comment, zt.Comment)
	}
	if len(z.File) != len(zt.File) {
		t.Errorf("%s: file count=%d, want %d", zt.Name, len(z.File), len(zt.File))
	}

	// test read of each file
	for i, ft := range zt.File {
		readTestFile(t, ft, z.File[i])
	}

	// test simultaneous reads
	n := 0
	done := make(chan bool)
	for i := 0; i < 5; i++ {
		for j, ft := range zt.File {
			go func(j int, ft ZipTestFile) {
				readTestFile(t, ft, z.File[j])
				done <- true
			}(j, ft)
			n++
		}
	}
	for ; n > 0; n-- {
		<-done
	}

	// test invalid checksum
	if !z.File[0].hasDataDescriptor() { // skip test when crc32 in dd
		z.File[0].CRC32++ // invalidate
		r, err := z.File[0].Open()
		if err != nil {
			t.Error(err)
			return
		}
		var b bytes.Buffer
		_, err = io.Copy(&b, r)
		if err != ChecksumError {
			t.Errorf("%s: copy error=%v, want %v", z.File[0].Name, err, ChecksumError)
		}
	}
}

func readTestFile(t *testing.T, ft ZipTestFile, f *File) {
	if f.Name != ft.Name {
		t.Errorf("name=%q, want %q", f.Name, ft.Name)
	}

	if ft.Mtime != "" {
		mtime, err := time.Parse("01-02-06 15:04:05", ft.Mtime)
		if err != nil {
			t.Error(err)
			return
		}
		if ft := f.ModTime(); !ft.Equal(mtime) {
			t.Errorf("%s: mtime=%s, want %s", f.Name, ft, mtime)
		}
	}

	testFileMode(t, f, ft.Mode)

	size0 := f.UncompressedSize

	var b bytes.Buffer
	r, err := f.Open()
	if err != nil {
		t.Error(err)
		return
	}

	if size1 := f.UncompressedSize; size0 != size1 {
		t.Errorf("file %q changed f.UncompressedSize from %d to %d", f.Name, size0, size1)
	}

	_, err = io.Copy(&b, r)
	if err != nil {
		t.Error(err)
		return
	}
	r.Close()

	var c []byte
	if ft.Content != nil {
		c = ft.Content
	} else if c, err = ioutil.ReadFile("testdata/" + ft.File); err != nil {
		t.Error(err)
		return
	}

	if b.Len() != len(c) {
		t.Errorf("%s: len=%d, want %d", f.Name, b.Len(), len(c))
		return
	}

	for i, b := range b.Bytes() {
		if b != c[i] {
			t.Errorf("%s: content[%d]=%q want %q", f.Name, i, b, c[i])
			return
		}
	}
}

func testFileMode(t *testing.T, f *File, want os.FileMode) {
	mode, err := f.Mode()
	if want == 0 {
		if err == nil {
			t.Errorf("%s mode: got %v, want none", f.Name, mode)
		}
	} else if err != nil {
		t.Errorf("%s mode: %s", f.Name, err)
	} else if mode != want {
		t.Errorf("%s mode: want %v, got %v", f.Name, want, mode)
	}
}

func TestInvalidFiles(t *testing.T) {
	const size = 1024 * 70 // 70kb
	b := make([]byte, size)

	// zeroes
	_, err := NewReader(sliceReaderAt(b), size)
	if err != FormatError {
		t.Errorf("zeroes: error=%v, want %v", err, FormatError)
	}

	// repeated directoryEndSignatures
	sig := make([]byte, 4)
	binary.LittleEndian.PutUint32(sig, directoryEndSignature)
	for i := 0; i < size-4; i += 4 {
		copy(b[i:i+4], sig)
	}
	_, err = NewReader(sliceReaderAt(b), size)
	if err != FormatError {
		t.Errorf("sigs: error=%v, want %v", err, FormatError)
	}
}

type sliceReaderAt []byte

func (r sliceReaderAt) ReadAt(b []byte, off int64) (int, error) {
	copy(b, r[int(off):int(off)+len(b)])
	return len(b), nil
}
<|endoftext|>


<commit_before>package asset

import (
	"bytes"
	"fmt"
	"image"
	"image/color"

	// for decoding of different file types
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"

	"github.com/goxjs/gl"
)

// LoadTexture from local assets. Handles jpg, png, and static gifs.
func LoadTexture(path string) (gl.Texture, error) {
	// based on https://developer.mozilla.org/en-US/docs/Web/API/WebGL_API/Tutorial/Using_textures_in_WebGL and https://golang.org/pkg/image/

	fileData, err := loadFile(path)
	if err != nil {
		return gl.Texture{}, err
	}

	img, _, err := image.Decode(bytes.NewBuffer(fileData))
	if err != nil {
		return gl.Texture{}, err
	}
	bounds := img.Bounds()
	width, height := bounds.Dx(), bounds.Dy()

	// Get raw RGBA pixels
	var data []uint8
	// Image checking from https://github.com/go-gl-legacy/glh/blob/master/texture.go
	switch trueim := img.(type) {
	case *image.RGBA:
		data = trueim.Pix
	case *image.NRGBA: // NRGBA is non-premultiplied RGBA. RGBA evidently is supposed to multiply the alpha value by the other colors, so NRGBA of (1, 0.5, 0, 0.5) is RGBA of (0.5, 0.25, 0. 0.5)
		data = trueim.Pix
	case *image.YCbCr:
		data = ycbCrToRGBA(trueim).Pix
	default:
		// TODO: This catch-all may work, but I haven't tested it enough.
		// copy := image.NewRGBA(trueim.Bounds())
		// draw.Draw(copy, trueim.Bounds(), trueim, image.Pt(0, 0), draw.Src)
		return gl.Texture{}, fmt.Errorf("unsupported texture format %T", img)
	}

	// Need to flip the image vertically since OpenGL considers 0,0 to be the top left corner.
	// Note width*4 since the data array consists of R,G,B,A values.
	if err := flipYCoords(data, width*4); err != nil {
		return gl.Texture{}, err
	}
	return LoadTextureData(width, height, data), nil
}

// LoadTextureData takes raw RGBA image data and puts it into a texture unit on the GPU.
// It's up to the caller to delete the texture buffer using gl.DeleteTexture(texture) when it's no longer needed.
func LoadTextureData(width, height int, data []uint8) gl.Texture {
	// gl.Enable(gl.TEXTURE_2D) // some sources says this is needed, but it doesn't seem to be. In fact, it gives an "invalid capability" message in webgl.

	texture := gl.CreateTexture()
	gl.BindTexture(gl.TEXTURE_2D, texture)
	// NOTE: gl.FLOAT isn't enabled for texture data types unless gl.getExtension('OES_texture_float'); is set, so just use gl.UNSIGNED_BYTE
	//   See http://stackoverflow.com/questions/23124597/storing-floats-in-a-texture-in-opengl-es  http://stackoverflow.com/questions/22666556/webgl-texture-creation-trouble
	gl.TexImage2D(gl.TEXTURE_2D, 0, width, height, gl.RGBA, gl.UNSIGNED_BYTE, data)
	gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR)
	gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR_MIPMAP_LINEAR)
	gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE)
	gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE)
	gl.GenerateMipmap(gl.TEXTURE_2D)
	gl.BindTexture(gl.TEXTURE_2D, gl.Texture{}) // bind to "null" to prevent using the wrong texture by mistake.
	return texture
}

func ycbCrToRGBA(img *image.YCbCr) *image.RGBA {
	b := img.Bounds()
	rgba := image.NewRGBA(b)
	for y := b.Min.Y; y < b.Max.Y; y++ {
		for x := b.Min.X; x < b.Max.X; x++ {
			r, g, b, a := img.At(x, y).RGBA()
			c := color.RGBA{uint8(r >> 8), uint8(g >> 8), uint8(b >> 8), uint8(a >> 8)}
			rgba.SetRGBA(x, y, c)
		}
	}
	return rgba
}

// Takes a flattened 2D array and the width of the rows.
// Modifies the values such that if the original array were an image, it would now appear upside down.
func flipYCoords(data []uint8, width int) error {
	if len(data)%width != 0 {
		return fmt.Errorf("expected flattened 2d array, got uneven row length: len %% width == %v", len(data)%width)
	}
	height := len(data) / width
	for row := 0; row < height/2; row++ {
		for col := 0; col < width; col++ {
			temp := data[col+row*width]
			data[col+row*width] = data[col+(height-1-row)*width]
			data[col+(height-1-row)*width] = temp
		}
	}
	return nil
}
<commit_msg>Improve image loading<commit_after>package asset

import (
	"bytes"
	"fmt"
	"image"
	"image/draw"

	// for decoding of different file types
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"

	"github.com/goxjs/gl"
)

// LoadTexture from local assets. Handles jpg, png, and static gifs.
func LoadTexture(path string) (gl.Texture, error) {
	// based on https://developer.mozilla.org/en-US/docs/Web/API/WebGL_API/Tutorial/Using_textures_in_WebGL and https://golang.org/pkg/image/

	fileData, err := loadFile(path)
	if err != nil {
		return gl.Texture{}, err
	}

	img, _, err := image.Decode(bytes.NewBuffer(fileData))
	if err != nil {
		return gl.Texture{}, err
	}
	bounds := img.Bounds()
	width, height := bounds.Dx(), bounds.Dy()

	// Get raw RGBA pixels by drawing the image into an NRGBA image. This is necessary to deal with different
	// image formats that aren't decoded into a pixel array. For example, jpeg compressed images are read in in a way
	// that mimics their encoding, and due to the way they are compressed, you can't get pixel values easily.
	// By drawing the decoded image out as NRGBA, we are guaranteed to get something we can deal with.
	// Note that this is wasteful for images that are already read in as RGBA or NRGBA, but it's a one time cost
	// and shouldn't be an issue.
	newimg := image.NewNRGBA(image.Rect(0, 0, width, height))
	draw.Draw(newimg, bounds, img, bounds.Min, draw.Src)
	data := newimg.Pix

	// Need to flip the image vertically since OpenGL considers 0,0 to be the top left corner.
	// Note width*4 since the data array consists of R,G,B,A values.
	if err := flipYCoords(data, width*4); err != nil {
		return gl.Texture{}, err
	}
	return LoadTextureData(width, height, data), nil
}

// LoadTextureData takes raw RGBA image data and puts it into a texture unit on the GPU.
// It's up to the caller to delete the texture buffer using gl.DeleteTexture(texture) when it's no longer needed.
func LoadTextureData(width, height int, data []uint8) gl.Texture {
	// gl.Enable(gl.TEXTURE_2D) // some sources says this is needed, but it doesn't seem to be. In fact, it gives an "invalid capability" message in webgl.

	texture := gl.CreateTexture()
	gl.BindTexture(gl.TEXTURE_2D, texture)
	// NOTE: gl.FLOAT isn't enabled for texture data types unless gl.getExtension('OES_texture_float'); is set, so just use gl.UNSIGNED_BYTE
	//   See http://stackoverflow.com/questions/23124597/storing-floats-in-a-texture-in-opengl-es  http://stackoverflow.com/questions/22666556/webgl-texture-creation-trouble
	gl.TexImage2D(gl.TEXTURE_2D, 0, width, height, gl.RGBA, gl.UNSIGNED_BYTE, data)
	gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR)
	gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR_MIPMAP_LINEAR)
	gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE)
	gl.TexParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE)
	gl.GenerateMipmap(gl.TEXTURE_2D)
	gl.BindTexture(gl.TEXTURE_2D, gl.Texture{}) // bind to "null" to prevent using the wrong texture by mistake.
	return texture
}

// Takes a flattened 2D array and the width of the rows.
// Modifies the values such that if the original array were an image, it would now appear upside down.
func flipYCoords(data []uint8, width int) error {
	if len(data)%width != 0 {
		return fmt.Errorf("expected flattened 2d array, got uneven row length: len %% width == %v", len(data)%width)
	}
	height := len(data) / width
	for row := 0; row < height/2; row++ {
		for col := 0; col < width; col++ {
			temp := data[col+row*width]
			data[col+row*width] = data[col+(height-1-row)*width]
			data[col+(height-1-row)*width] = temp
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package lint

import (
	"bytes"
	"io/ioutil"
	"os/exec"
	"regexp"
	"strings"

	"github.com/ValeLint/vale/core"
	"github.com/russross/blackfriday"
	"golang.org/x/net/html"
)

// reCodeBlock is used to convert Sphinx-style code directives to the regular
// `::` for rst2html.
var reCodeBlock = regexp.MustCompile(`.. (?:raw|code(?:-block)?):: (\w+)`)

// Blackfriday configuration.
var commonHTMLFlags = 0 | blackfriday.HTML_USE_XHTML
var commonExtensions = 0 |
	blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
	blackfriday.EXTENSION_TABLES |
	blackfriday.EXTENSION_FENCED_CODE
var renderer = blackfriday.HtmlRenderer(commonHTMLFlags, "", "")
var options = blackfriday.Options{Extensions: commonExtensions}

func (l Linter) lintHTMLTokens(f *core.File, rawBytes []byte, fBytes []byte, offset int) {
	var txt, attr string
	var tokt html.TokenType
	var tok html.Token
	var inBlock, skip, isHeading bool

	ctx := core.PrepText(string(rawBytes))
	heading := regexp.MustCompile(`^h\d$`)
	skipTags := []string{"script", "style", "pre", "code", "tt"}
	skipClasses := []string{"LaTeX"}
	lines := strings.Count(ctx, "\n") + 1 + offset

	tokens := html.NewTokenizer(bytes.NewReader(fBytes))
	for {
		tokt = tokens.Next()
		tok = tokens.Token()
		txt = html.UnescapeString(strings.TrimSpace(tok.Data))
		skip = core.StringInSlice(txt, skipTags) || core.StringInSlice(attr, skipClasses)
		if tokt == html.ErrorToken {
			break
		} else if tokt == html.StartTagToken && skip {
			inBlock = true
		} else if skip && inBlock {
			inBlock = false
		} else if tokt == html.StartTagToken && heading.MatchString(txt) {
			isHeading = true
		} else if tokt == html.EndTagToken && isHeading {
			isHeading = false
		} else if tokt == html.TextToken && isHeading && !inBlock && txt != "" {
			l.lintText(f, NewBlock(ctx, txt, "heading"+f.RealExt), lines, 0)
		} else if tokt == html.TextToken && !inBlock && !skip {
			l.lintProse(f, ctx, txt, lines, 0)
		}
		attr = getAttribute(tok, "class")
		ctx = updateCtx(ctx, txt, tokt, tok)
	}
}

func getAttribute(tok html.Token, key string) string {
	for _, attr := range tok.Attr {
		if attr.Key == key {
			return attr.Val
		}
	}
	return ""
}

func updateCtx(ctx string, txt string, tokt html.TokenType, tok html.Token) string {
	if tok.Data == "img" || tok.Data == "a" {
		for _, a := range tok.Attr {
			if a.Key == "alt" || a.Key == "href" {
				ctx = core.Substitute(ctx, a.Val, "*")
			}
		}
	} else if tokt == html.TextToken {
		for _, s := range strings.Split(txt, "\n") {
			ctx = core.Substitute(ctx, s, "*")
		}
	}
	return ctx
}

func (l Linter) lintHTML(f *core.File) {
	b, err := ioutil.ReadFile(f.Path)
	if !core.CheckError(err, f.Path) {
		return
	}
	l.lintHTMLTokens(f, b, b, 0)
}

func (l Linter) lintMarkdown(f *core.File) {
	b, err := ioutil.ReadFile(f.Path)
	if !core.CheckError(err, f.Path) {
		return
	}
	l.lintHTMLTokens(f, b, blackfriday.MarkdownOptions(b, renderer, options), 0)
}

func (l Linter) lintRST(f *core.File, python string, rst2html string) {
	var out bytes.Buffer
	b, err := ioutil.ReadFile(f.Path)
	if !core.CheckError(err, f.Path) {
		return
	}
	cmd := exec.Command(
		python, rst2html, "--quiet", "--halt=5", "--link-stylesheet")
	cmd.Stdin = bytes.NewReader(reCodeBlock.ReplaceAll(b, []byte("::")))
	cmd.Stdout = &out
	if core.CheckError(cmd.Run(), f.Path) {
		l.lintHTMLTokens(f, b, out.Bytes(), 0)
	}
}

func (l Linter) lintADoc(f *core.File, asciidoctor string) {
	var out bytes.Buffer
	b, err := ioutil.ReadFile(f.Path)
	if !core.CheckError(err, f.Path) {
		return
	}
	cmd := exec.Command(asciidoctor, "--no-header-footer", "--safe", "-")
	cmd.Stdin = bytes.NewReader(b)
	cmd.Stdout = &out
	if core.CheckError(cmd.Run(), f.Path) {
		l.lintHTMLTokens(f, b, out.Bytes(), 0)
	}
}

func (l Linter) lintLaTeX(f *core.File, pandoc string) {
	var out bytes.Buffer
	b, err := ioutil.ReadFile(f.Path)
	if !core.CheckError(err, f.Path) {
		return
	}

	start := bytes.Index(b, []byte(`\begin{document}`))
	lines := 0
	if start >= 0 {
		lines = bytes.Count(b[:start], []byte("\n"))
		b = b[start:]
	}

	cmd := exec.Command(pandoc, "--listings", "--latexmathml", f.Path)
	cmd.Stdout = &out
	if core.CheckError(cmd.Run(), f.Path) {
		l.lintHTMLTokens(f, b, out.Bytes(), lines)
	}
}
<commit_msg>refactor: prep text before calling `updateCtx`<commit_after>package lint

import (
	"bytes"
	"io/ioutil"
	"os/exec"
	"regexp"
	"strings"

	"github.com/ValeLint/vale/core"
	"github.com/russross/blackfriday"
	"golang.org/x/net/html"
)

// reCodeBlock is used to convert Sphinx-style code directives to the regular
// `::` for rst2html.
var reCodeBlock = regexp.MustCompile(`.. (?:raw|code(?:-block)?):: (\w+)`)

// Blackfriday configuration.
var commonHTMLFlags = 0 | blackfriday.HTML_USE_XHTML
var commonExtensions = 0 |
	blackfriday.EXTENSION_NO_INTRA_EMPHASIS |
	blackfriday.EXTENSION_TABLES |
	blackfriday.EXTENSION_FENCED_CODE
var renderer = blackfriday.HtmlRenderer(commonHTMLFlags, "", "")
var options = blackfriday.Options{Extensions: commonExtensions}

func (l Linter) lintHTMLTokens(f *core.File, rawBytes []byte, fBytes []byte, offset int) {
	var txt, attr string
	var tokt html.TokenType
	var tok html.Token
	var inBlock, skip, isHeading bool

	ctx := core.PrepText(string(rawBytes))
	heading := regexp.MustCompile(`^h\d$`)
	skipTags := []string{"script", "style", "pre", "code", "tt"}
	skipClasses := []string{"LaTeX"}
	lines := strings.Count(ctx, "\n") + 1 + offset

	tokens := html.NewTokenizer(bytes.NewReader(fBytes))
	for {
		tokt = tokens.Next()
		tok = tokens.Token()
		txt = core.PrepText(html.UnescapeString(strings.TrimSpace(tok.Data)))
		skip = core.StringInSlice(txt, skipTags) || core.StringInSlice(attr, skipClasses)
		if tokt == html.ErrorToken {
			break
		} else if tokt == html.StartTagToken && skip {
			inBlock = true
		} else if skip && inBlock {
			inBlock = false
		} else if tokt == html.StartTagToken && heading.MatchString(txt) {
			isHeading = true
		} else if tokt == html.EndTagToken && isHeading {
			isHeading = false
		} else if tokt == html.TextToken && isHeading && !inBlock && txt != "" {
			l.lintText(f, NewBlock(ctx, txt, "heading"+f.RealExt), lines, 0)
		} else if tokt == html.TextToken && !inBlock && !skip {
			l.lintProse(f, ctx, txt, lines, 0)
		}
		attr = getAttribute(tok, "class")
		ctx = updateCtx(ctx, txt, tokt, tok)
	}
}

func getAttribute(tok html.Token, key string) string {
	for _, attr := range tok.Attr {
		if attr.Key == key {
			return attr.Val
		}
	}
	return ""
}

func updateCtx(ctx string, txt string, tokt html.TokenType, tok html.Token) string {
	if tok.Data == "img" || tok.Data == "a" {
		for _, a := range tok.Attr {
			if a.Key == "alt" || a.Key == "href" {
				ctx = core.Substitute(ctx, a.Val, "*")
			}
		}
	} else if tokt == html.TextToken && txt != "" {
		for _, s := range strings.Split(txt, "\n") {
			ctx = core.Substitute(ctx, s, "*")
		}
	}
	return ctx
}

func (l Linter) lintHTML(f *core.File) {
	b, err := ioutil.ReadFile(f.Path)
	if !core.CheckError(err, f.Path) {
		return
	}
	l.lintHTMLTokens(f, b, b, 0)
}

func (l Linter) lintMarkdown(f *core.File) {
	b, err := ioutil.ReadFile(f.Path)
	if !core.CheckError(err, f.Path) {
		return
	}
	l.lintHTMLTokens(f, b, blackfriday.MarkdownOptions(b, renderer, options), 0)
}

func (l Linter) lintRST(f *core.File, python string, rst2html string) {
	var out bytes.Buffer
	b, err := ioutil.ReadFile(f.Path)
	if !core.CheckError(err, f.Path) {
		return
	}
	cmd := exec.Command(
		python, rst2html, "--quiet", "--halt=5", "--link-stylesheet")
	cmd.Stdin = bytes.NewReader(reCodeBlock.ReplaceAll(b, []byte("::")))
	cmd.Stdout = &out
	if core.CheckError(cmd.Run(), f.Path) {
		l.lintHTMLTokens(f, b, out.Bytes(), 0)
	}
}

func (l Linter) lintADoc(f *core.File, asciidoctor string) {
	var out bytes.Buffer
	b, err := ioutil.ReadFile(f.Path)
	if !core.CheckError(err, f.Path) {
		return
	}
	cmd := exec.Command(asciidoctor, "--no-header-footer", "--safe", "-")
	cmd.Stdin = bytes.NewReader(b)
	cmd.Stdout = &out
	if core.CheckError(cmd.Run(), f.Path) {
		l.lintHTMLTokens(f, b, out.Bytes(), 0)
	}
}

func (l Linter) lintLaTeX(f *core.File, pandoc string) {
	var out bytes.Buffer
	b, err := ioutil.ReadFile(f.Path)
	if !core.CheckError(err, f.Path) {
		return
	}

	start := bytes.Index(b, []byte(`\begin{document}`))
	lines := 0
	if start >= 0 {
		lines = bytes.Count(b[:start], []byte("\n"))
		b = b[start:]
	}

	cmd := exec.Command(pandoc, "--listings", "--latexmathml", f.Path)
	cmd.Stdout = &out
	if core.CheckError(cmd.Run(), f.Path) {
		l.lintHTMLTokens(f, b, out.Bytes(), lines)
	}
}
<|endoftext|>