From 73df946d56c74384511a194dd01dbe099584fd1a Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 28 Apr 2024 15:14:23 +0200 Subject: Adding upstream version 1.16.10. Signed-off-by: Daniel Baumann --- src/cmd/go/internal/renameio/renameio.go | 94 +++++++++++++++ src/cmd/go/internal/renameio/renameio_test.go | 160 ++++++++++++++++++++++++++ src/cmd/go/internal/renameio/umask_test.go | 42 +++++++ 3 files changed, 296 insertions(+) create mode 100644 src/cmd/go/internal/renameio/renameio.go create mode 100644 src/cmd/go/internal/renameio/renameio_test.go create mode 100644 src/cmd/go/internal/renameio/umask_test.go (limited to 'src/cmd/go/internal/renameio') diff --git a/src/cmd/go/internal/renameio/renameio.go b/src/cmd/go/internal/renameio/renameio.go new file mode 100644 index 0000000..9788171 --- /dev/null +++ b/src/cmd/go/internal/renameio/renameio.go @@ -0,0 +1,94 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package renameio writes files atomically by renaming temporary files. +package renameio + +import ( + "bytes" + "io" + "io/fs" + "math/rand" + "os" + "path/filepath" + "strconv" + + "cmd/go/internal/robustio" +) + +const patternSuffix = ".tmp" + +// Pattern returns a glob pattern that matches the unrenamed temporary files +// created when writing to filename. +func Pattern(filename string) string { + return filepath.Join(filepath.Dir(filename), filepath.Base(filename)+patternSuffix) +} + +// WriteFile is like os.WriteFile, but first writes data to an arbitrary +// file in the same directory as filename, then renames it atomically to the +// final name. +// +// That ensures that the final location, if it exists, is always a complete file. +func WriteFile(filename string, data []byte, perm fs.FileMode) (err error) { + return WriteToFile(filename, bytes.NewReader(data), perm) +} + +// WriteToFile is a variant of WriteFile that accepts the data as an io.Reader +// instead of a slice. +func WriteToFile(filename string, data io.Reader, perm fs.FileMode) (err error) { + f, err := tempFile(filepath.Dir(filename), filepath.Base(filename), perm) + if err != nil { + return err + } + defer func() { + // Only call os.Remove on f.Name() if we failed to rename it: otherwise, + // some other process may have created a new file with the same name after + // that. + if err != nil { + f.Close() + os.Remove(f.Name()) + } + }() + + if _, err := io.Copy(f, data); err != nil { + return err + } + // Sync the file before renaming it: otherwise, after a crash the reader may + // observe a 0-length file instead of the actual contents. + // See https://golang.org/issue/22397#issuecomment-380831736. + if err := f.Sync(); err != nil { + return err + } + if err := f.Close(); err != nil { + return err + } + + return robustio.Rename(f.Name(), filename) +} + +// ReadFile is like os.ReadFile, but on Windows retries spurious errors that +// may occur if the file is concurrently replaced. +// +// Errors are classified heuristically and retries are bounded, so even this +// function may occasionally return a spurious error on Windows. +// If so, the error will likely wrap one of: +// - syscall.ERROR_ACCESS_DENIED +// - syscall.ERROR_FILE_NOT_FOUND +// - internal/syscall/windows.ERROR_SHARING_VIOLATION +func ReadFile(filename string) ([]byte, error) { + return robustio.ReadFile(filename) +} + +// tempFile creates a new temporary file with given permission bits. +func tempFile(dir, prefix string, perm fs.FileMode) (f *os.File, err error) { + for i := 0; i < 10000; i++ { + name := filepath.Join(dir, prefix+strconv.Itoa(rand.Intn(1000000000))+patternSuffix) + f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, perm) + if os.IsExist(err) { + continue + } + break + } + return +} diff --git a/src/cmd/go/internal/renameio/renameio_test.go b/src/cmd/go/internal/renameio/renameio_test.go new file mode 100644 index 0000000..5b2ed83 --- /dev/null +++ b/src/cmd/go/internal/renameio/renameio_test.go @@ -0,0 +1,160 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !plan9 + +package renameio + +import ( + "encoding/binary" + "errors" + "internal/testenv" + "math/rand" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "sync/atomic" + "syscall" + "testing" + "time" + + "cmd/go/internal/robustio" +) + +func TestConcurrentReadsAndWrites(t *testing.T) { + if runtime.GOOS == "darwin" && strings.HasSuffix(testenv.Builder(), "-10_14") { + testenv.SkipFlaky(t, 33041) + } + + dir, err := os.MkdirTemp("", "renameio") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + path := filepath.Join(dir, "blob.bin") + + const chunkWords = 8 << 10 + buf := make([]byte, 2*chunkWords*8) + for i := uint64(0); i < 2*chunkWords; i++ { + binary.LittleEndian.PutUint64(buf[i*8:], i) + } + + var attempts int64 = 128 + if !testing.Short() { + attempts *= 16 + } + const parallel = 32 + + var sem = make(chan bool, parallel) + + var ( + writeSuccesses, readSuccesses int64 // atomic + writeErrnoSeen, readErrnoSeen sync.Map + ) + + for n := attempts; n > 0; n-- { + sem <- true + go func() { + defer func() { <-sem }() + + time.Sleep(time.Duration(rand.Intn(100)) * time.Microsecond) + offset := rand.Intn(chunkWords) + chunk := buf[offset*8 : (offset+chunkWords)*8] + if err := WriteFile(path, chunk, 0666); err == nil { + atomic.AddInt64(&writeSuccesses, 1) + } else if robustio.IsEphemeralError(err) { + var ( + errno syscall.Errno + dup bool + ) + if errors.As(err, &errno) { + _, dup = writeErrnoSeen.LoadOrStore(errno, true) + } + if !dup { + t.Logf("ephemeral error: %v", err) + } + } else { + t.Errorf("unexpected error: %v", err) + } + + time.Sleep(time.Duration(rand.Intn(100)) * time.Microsecond) + data, err := ReadFile(path) + if err == nil { + atomic.AddInt64(&readSuccesses, 1) + } else if robustio.IsEphemeralError(err) { + var ( + errno syscall.Errno + dup bool + ) + if errors.As(err, &errno) { + _, dup = readErrnoSeen.LoadOrStore(errno, true) + } + if !dup { + t.Logf("ephemeral error: %v", err) + } + return + } else { + t.Errorf("unexpected error: %v", err) + return + } + + if len(data) != 8*chunkWords { + t.Errorf("read %d bytes, but each write is a %d-byte file", len(data), 8*chunkWords) + return + } + + u := binary.LittleEndian.Uint64(data) + for i := 1; i < chunkWords; i++ { + next := binary.LittleEndian.Uint64(data[i*8:]) + if next != u+1 { + t.Errorf("wrote sequential integers, but read integer out of sequence at offset %d", i) + return + } + u = next + } + }() + } + + for n := parallel; n > 0; n-- { + sem <- true + } + + var minWriteSuccesses int64 = attempts + if runtime.GOOS == "windows" { + // Windows produces frequent "Access is denied" errors under heavy rename load. + // As long as those are the only errors and *some* of the writes succeed, we're happy. + minWriteSuccesses = attempts / 4 + } + + if writeSuccesses < minWriteSuccesses { + t.Errorf("%d (of %d) writes succeeded; want ≥ %d", writeSuccesses, attempts, minWriteSuccesses) + } else { + t.Logf("%d (of %d) writes succeeded (ok: ≥ %d)", writeSuccesses, attempts, minWriteSuccesses) + } + + var minReadSuccesses int64 = attempts + + switch runtime.GOOS { + case "windows": + // Windows produces frequent "Access is denied" errors under heavy rename load. + // As long as those are the only errors and *some* of the reads succeed, we're happy. + minReadSuccesses = attempts / 4 + + case "darwin", "ios": + // The filesystem on certain versions of macOS (10.14) and iOS (affected + // versions TBD) occasionally fail with "no such file or directory" errors. + // See https://golang.org/issue/33041 and https://golang.org/issue/42066. + // The flake rate is fairly low, so ensure that at least 75% of attempts + // succeed. + minReadSuccesses = attempts - (attempts / 4) + } + + if readSuccesses < minReadSuccesses { + t.Errorf("%d (of %d) reads succeeded; want ≥ %d", readSuccesses, attempts, minReadSuccesses) + } else { + t.Logf("%d (of %d) reads succeeded (ok: ≥ %d)", readSuccesses, attempts, minReadSuccesses) + } +} diff --git a/src/cmd/go/internal/renameio/umask_test.go b/src/cmd/go/internal/renameio/umask_test.go new file mode 100644 index 0000000..65e4fa5 --- /dev/null +++ b/src/cmd/go/internal/renameio/umask_test.go @@ -0,0 +1,42 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !plan9,!windows,!js + +package renameio + +import ( + "io/fs" + "os" + "path/filepath" + "syscall" + "testing" +) + +func TestWriteFileModeAppliesUmask(t *testing.T) { + dir, err := os.MkdirTemp("", "renameio") + if err != nil { + t.Fatalf("Failed to create temporary directory: %v", err) + } + defer os.RemoveAll(dir) + + const mode = 0644 + const umask = 0007 + defer syscall.Umask(syscall.Umask(umask)) + + file := filepath.Join(dir, "testWrite") + err = WriteFile(file, []byte("go-build"), mode) + if err != nil { + t.Fatalf("Failed to write file: %v", err) + } + + fi, err := os.Stat(file) + if err != nil { + t.Fatalf("Stat %q (looking for mode %#o): %s", file, mode, err) + } + + if fi.Mode()&fs.ModePerm != 0640 { + t.Errorf("Stat %q: mode %#o want %#o", file, fi.Mode()&fs.ModePerm, 0640) + } +} -- cgit v1.2.3