352 lines
11 KiB
Diff
352 lines
11 KiB
Diff
upstream: https://github.com/kubernetes/kubernetes/pull/96115
|
|
|
|
From d45ba645a8f7b288284890a051c73bbae717da4b Mon Sep 17 00:00:00 2001
|
|
From: Natanael Copa <ncopa@mirantis.com>
|
|
Date: Wed, 30 Sep 2020 15:19:27 +0200
|
|
Subject: [PATCH] Get inodes and disk usage via pure go
|
|
|
|
Fix inode usage calculation to use filepath.Walk instead of executing an
|
|
external find. Also calculate the disk usage while at it so we also get
|
|
rid of the external dependency of `nice` and `du`. (#95172)
|
|
|
|
This is similar to what cadvisor does since commit
|
|
https://github.com/google/cadvisor/commit/046818d64c0af62a4c5037583b467296bb68626d
|
|
|
|
This solves three problems:
|
|
- Counts number of inodes correct when there are hardlinks (#96114)
|
|
- Makes kubelet work without GNU findutils (#95186)
|
|
- Makes kubelet work without GNU coreutils (#95172)
|
|
---
|
|
pkg/volume/metrics_du.go | 26 ++---
|
|
pkg/volume/util/fs/fs.go | 132 +++++++++++++++-----------
|
|
pkg/volume/util/fs/fs_unsupported.go | 17 ++--
|
|
pkg/volume/util/fs/fs_windows.go | 30 ++----
|
|
pkg/volume/util/fs/fs_windows_test.go | 7 +-
|
|
5 files changed, 108 insertions(+), 104 deletions(-)
|
|
|
|
diff --git a/pkg/volume/metrics_du.go b/pkg/volume/metrics_du.go
|
|
index 1491c30de76d83..f080fac2b33bce 100644
|
|
--- a/pkg/volume/metrics_du.go
|
|
+++ b/pkg/volume/metrics_du.go
|
|
@@ -46,12 +46,7 @@ func (md *metricsDu) GetMetrics() (*Metrics, error) {
|
|
return metrics, NewNoPathDefinedError()
|
|
}
|
|
|
|
- err := md.runDiskUsage(metrics)
|
|
- if err != nil {
|
|
- return metrics, err
|
|
- }
|
|
-
|
|
- err = md.runFind(metrics)
|
|
+ err := md.getDiskUsage(metrics)
|
|
if err != nil {
|
|
return metrics, err
|
|
}
|
|
@@ -64,23 +59,14 @@ func (md *metricsDu) GetMetrics() (*Metrics, error) {
|
|
return metrics, nil
|
|
}
|
|
|
|
-// runDiskUsage gets disk usage of md.path and writes the results to metrics.Used
|
|
-func (md *metricsDu) runDiskUsage(metrics *Metrics) error {
|
|
- used, err := fs.DiskUsage(md.path)
|
|
- if err != nil {
|
|
- return err
|
|
- }
|
|
- metrics.Used = used
|
|
- return nil
|
|
-}
|
|
-
|
|
-// runFind executes the "find" command and writes the results to metrics.InodesUsed
|
|
-func (md *metricsDu) runFind(metrics *Metrics) error {
|
|
- inodesUsed, err := fs.Find(md.path)
|
|
+// getDiskUsage writes metrics.Used and metric.InodesUsed from fs.DiskUsage
|
|
+func (md *metricsDu) getDiskUsage(metrics *Metrics) error {
|
|
+ usage, err := fs.DiskUsage(md.path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
- metrics.InodesUsed = resource.NewQuantity(inodesUsed, resource.BinarySI)
|
|
+ metrics.Used = resource.NewQuantity(usage.Bytes, resource.BinarySI)
|
|
+ metrics.InodesUsed = resource.NewQuantity(usage.Inodes, resource.BinarySI)
|
|
return nil
|
|
}
|
|
|
|
diff --git a/pkg/volume/util/fs/fs.go b/pkg/volume/util/fs/fs.go
|
|
index a6114f259a6637..77adf3747856ea 100644
|
|
--- a/pkg/volume/util/fs/fs.go
|
|
+++ b/pkg/volume/util/fs/fs.go
|
|
@@ -19,17 +19,21 @@ limitations under the License.
|
|
package fs
|
|
|
|
import (
|
|
- "bytes"
|
|
"fmt"
|
|
- "os/exec"
|
|
- "strings"
|
|
+ "os"
|
|
+ "path/filepath"
|
|
+ "syscall"
|
|
|
|
"golang.org/x/sys/unix"
|
|
|
|
- "k8s.io/apimachinery/pkg/api/resource"
|
|
"k8s.io/kubernetes/pkg/volume/util/fsquota"
|
|
)
|
|
|
|
+type UsageInfo struct {
|
|
+ Bytes int64
|
|
+ Inodes int64
|
|
+}
|
|
+
|
|
// Info linux returns (available bytes, byte capacity, byte usage, total inodes, inodes free, inode usage, error)
|
|
// for the filesystem that path resides upon.
|
|
func Info(path string) (int64, int64, int64, int64, int64, int64, error) {
|
|
@@ -55,63 +59,83 @@ func Info(path string) (int64, int64, int64, int64, int64, int64, error) {
|
|
return available, capacity, usage, inodes, inodesFree, inodesUsed, nil
|
|
}
|
|
|
|
-// DiskUsage gets disk usage of specified path.
|
|
-func DiskUsage(path string) (*resource.Quantity, error) {
|
|
- // First check whether the quota system knows about this directory
|
|
- // A nil quantity with no error means that the path does not support quotas
|
|
- // and we should use other mechanisms.
|
|
- data, err := fsquota.GetConsumption(path)
|
|
- if data != nil {
|
|
- return data, nil
|
|
- } else if err != nil {
|
|
- return nil, fmt.Errorf("unable to retrieve disk consumption via quota for %s: %v", path, err)
|
|
- }
|
|
- // Uses the same niceness level as cadvisor.fs does when running du
|
|
- // Uses -B 1 to always scale to a blocksize of 1 byte
|
|
- out, err := exec.Command("nice", "-n", "19", "du", "-x", "-s", "-B", "1", path).CombinedOutput()
|
|
- if err != nil {
|
|
- return nil, fmt.Errorf("failed command 'du' ($ nice -n 19 du -x -s -B 1) on path %s with error %v", path, err)
|
|
- }
|
|
- used, err := resource.ParseQuantity(strings.Fields(string(out))[0])
|
|
- if err != nil {
|
|
- return nil, fmt.Errorf("failed to parse 'du' output %s due to error %v", out, err)
|
|
- }
|
|
- used.Format = resource.BinarySI
|
|
- return &used, nil
|
|
-}
|
|
+// DiskUsage calculates the number of inodes and disk usage for a given directory
|
|
+func DiskUsage(path string) (UsageInfo, error) {
|
|
+ var usage UsageInfo
|
|
|
|
-// Find uses the equivalent of the command `find <path> -dev -printf '.' | wc -c` to count files and directories.
|
|
-// While this is not an exact measure of inodes used, it is a very good approximation.
|
|
-func Find(path string) (int64, error) {
|
|
if path == "" {
|
|
- return 0, fmt.Errorf("invalid directory")
|
|
+ return usage, fmt.Errorf("invalid directory")
|
|
}
|
|
+
|
|
// First check whether the quota system knows about this directory
|
|
- // A nil quantity with no error means that the path does not support quotas
|
|
- // and we should use other mechanisms.
|
|
- inodes, err := fsquota.GetInodes(path)
|
|
- if inodes != nil {
|
|
- return inodes.Value(), nil
|
|
- } else if err != nil {
|
|
- return 0, fmt.Errorf("unable to retrieve inode consumption via quota for %s: %v", path, err)
|
|
+ // A nil quantity or error means that the path does not support quotas
|
|
+ // or xfs_quota tool is missing and we should use other mechanisms.
|
|
+ consumption, _ := fsquota.GetConsumption(path)
|
|
+ if consumption != nil {
|
|
+ usage.Bytes = consumption.Value()
|
|
}
|
|
- var counter byteCounter
|
|
- var stderr bytes.Buffer
|
|
- findCmd := exec.Command("find", path, "-xdev", "-printf", ".")
|
|
- findCmd.Stdout, findCmd.Stderr = &counter, &stderr
|
|
- if err := findCmd.Start(); err != nil {
|
|
- return 0, fmt.Errorf("failed to exec cmd %v - %v; stderr: %v", findCmd.Args, err, stderr.String())
|
|
+
|
|
+ inodes, _ := fsquota.GetInodes(path)
|
|
+ if inodes != nil {
|
|
+ usage.Inodes = inodes.Value()
|
|
}
|
|
- if err := findCmd.Wait(); err != nil {
|
|
- return 0, fmt.Errorf("cmd %v failed. stderr: %s; err: %v", findCmd.Args, stderr.String(), err)
|
|
+
|
|
+ if inodes != nil && consumption != nil {
|
|
+ return usage, nil
|
|
}
|
|
- return counter.bytesWritten, nil
|
|
-}
|
|
|
|
-// Simple io.Writer implementation that counts how many bytes were written.
|
|
-type byteCounter struct{ bytesWritten int64 }
|
|
+ topLevelStat := &unix.Stat_t{}
|
|
+ err := unix.Stat(path, topLevelStat)
|
|
+ if err != nil {
|
|
+ return usage, err
|
|
+ }
|
|
|
|
-func (b *byteCounter) Write(p []byte) (int, error) {
|
|
- b.bytesWritten += int64(len(p))
|
|
- return len(p), nil
|
|
+ // dedupedInode stores inodes that could be duplicates (nlink > 1)
|
|
+ dedupedInodes := make(map[uint64]struct{})
|
|
+
|
|
+ err = filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
|
|
+ // ignore files that have been deleted after directory was read
|
|
+ if os.IsNotExist(err) {
|
|
+ return nil
|
|
+ }
|
|
+ if err != nil {
|
|
+ return fmt.Errorf("unable to count inodes for %s: %s", path, err)
|
|
+ }
|
|
+
|
|
+ // according to the docs, Sys can be nil
|
|
+ if info.Sys() == nil {
|
|
+ return fmt.Errorf("fileinfo Sys is nil")
|
|
+ }
|
|
+
|
|
+ s, ok := info.Sys().(*syscall.Stat_t)
|
|
+ if !ok {
|
|
+ return fmt.Errorf("unsupported fileinfo; could not convert to stat_t")
|
|
+ }
|
|
+
|
|
+ if s.Dev != topLevelStat.Dev {
|
|
+ // don't descend into directories on other devices
|
|
+ return filepath.SkipDir
|
|
+ }
|
|
+
|
|
+ // Dedupe hardlinks
|
|
+ if s.Nlink > 1 {
|
|
+ if _, ok := dedupedInodes[s.Ino]; !ok {
|
|
+ dedupedInodes[s.Ino] = struct{}{}
|
|
+ } else {
|
|
+ return nil
|
|
+ }
|
|
+ }
|
|
+
|
|
+ if consumption == nil {
|
|
+ usage.Bytes += int64(s.Blocks) * int64(512) // blocksize in bytes
|
|
+ }
|
|
+
|
|
+ if inodes == nil {
|
|
+ usage.Inodes++
|
|
+ }
|
|
+
|
|
+ return nil
|
|
+ })
|
|
+
|
|
+ return usage, err
|
|
}
|
|
diff --git a/pkg/volume/util/fs/fs_unsupported.go b/pkg/volume/util/fs/fs_unsupported.go
|
|
index 8cadf72bd53e79..6a098cb3f55f65 100644
|
|
--- a/pkg/volume/util/fs/fs_unsupported.go
|
|
+++ b/pkg/volume/util/fs/fs_unsupported.go
|
|
@@ -20,21 +20,20 @@ package fs
|
|
|
|
import (
|
|
"fmt"
|
|
-
|
|
- "k8s.io/apimachinery/pkg/api/resource"
|
|
)
|
|
|
|
+type UsageInfo struct {
|
|
+ Bytes int64
|
|
+ Inodes int64
|
|
+}
|
|
+
|
|
// Info unsupported returns 0 values for available and capacity and an error.
|
|
func Info(path string) (int64, int64, int64, int64, int64, int64, error) {
|
|
return 0, 0, 0, 0, 0, 0, fmt.Errorf("fsinfo not supported for this build")
|
|
}
|
|
|
|
// DiskUsage gets disk usage of specified path.
|
|
-func DiskUsage(path string) (*resource.Quantity, error) {
|
|
- return nil, fmt.Errorf("du not supported for this build")
|
|
-}
|
|
-
|
|
-// Find will always return zero since is on unsupported platform.
|
|
-func Find(path string) (int64, error) {
|
|
- return 0, fmt.Errorf("find not supported for this build")
|
|
+func DiskUsage(path string) (UsageInfo, error) {
|
|
+ var usage UsageInfo
|
|
+ return usage, fmt.Errorf("directory disk usage not supported for this build.")
|
|
}
|
|
diff --git a/pkg/volume/util/fs/fs_windows.go b/pkg/volume/util/fs/fs_windows.go
|
|
index 8d16eabcefeccb..356b2e223bff2d 100644
|
|
--- a/pkg/volume/util/fs/fs_windows.go
|
|
+++ b/pkg/volume/util/fs/fs_windows.go
|
|
@@ -26,8 +26,6 @@ import (
|
|
"unsafe"
|
|
|
|
"golang.org/x/sys/windows"
|
|
-
|
|
- "k8s.io/apimachinery/pkg/api/resource"
|
|
)
|
|
|
|
var (
|
|
@@ -35,6 +33,11 @@ var (
|
|
procGetDiskFreeSpaceEx = modkernel32.NewProc("GetDiskFreeSpaceExW")
|
|
)
|
|
|
|
+type UsageInfo struct {
|
|
+ Bytes int64
|
|
+ Inodes int64
|
|
+}
|
|
+
|
|
// Info returns (available bytes, byte capacity, byte usage, total inodes, inodes free, inode usage, error)
|
|
// for the filesystem that path resides upon.
|
|
func Info(path string) (int64, int64, int64, int64, int64, int64, error) {
|
|
@@ -64,28 +67,15 @@ func Info(path string) (int64, int64, int64, int64, int64, int64, error) {
|
|
}
|
|
|
|
// DiskUsage gets disk usage of specified path.
|
|
-func DiskUsage(path string) (*resource.Quantity, error) {
|
|
+func DiskUsage(path string) (UsageInfo, error) {
|
|
+ var usage UsageInfo
|
|
info, err := os.Lstat(path)
|
|
if err != nil {
|
|
- return nil, err
|
|
- }
|
|
-
|
|
- usage, err := diskUsage(path, info)
|
|
- if err != nil {
|
|
- return nil, err
|
|
+ return usage, err
|
|
}
|
|
|
|
- used, err := resource.ParseQuantity(fmt.Sprintf("%d", usage))
|
|
- if err != nil {
|
|
- return nil, fmt.Errorf("failed to parse fs usage %d due to %v", usage, err)
|
|
- }
|
|
- used.Format = resource.BinarySI
|
|
- return &used, nil
|
|
-}
|
|
-
|
|
-// Find will always return zero since inodes is not supported on Windows.
|
|
-func Find(path string) (int64, error) {
|
|
- return 0, nil
|
|
+ usage.Bytes, err = diskUsage(path, info)
|
|
+ return usage, err
|
|
}
|
|
|
|
func diskUsage(currPath string, info os.FileInfo) (int64, error) {
|
|
diff --git a/pkg/volume/util/fs/fs_windows_test.go b/pkg/volume/util/fs/fs_windows_test.go
|
|
index df57d82b548546..c9f69ba6941faa 100644
|
|
--- a/pkg/volume/util/fs/fs_windows_test.go
|
|
+++ b/pkg/volume/util/fs/fs_windows_test.go
|
|
@@ -67,10 +67,15 @@ func TestDiskUsage(t *testing.T) {
|
|
}
|
|
total := dirInfo1.Size() + dirInfo2.Size() + fileInfo1.Size() + fileInfo2.Size()
|
|
|
|
- size, err := DiskUsage(dir1)
|
|
+ usage, err := DiskUsage(dir1)
|
|
if err != nil {
|
|
t.Fatalf("TestDiskUsage failed: %s", err.Error())
|
|
}
|
|
+ size, err := resource.ParseQuantity(fmt.Sprintf("%d", usage.Bytes))
|
|
+ if err != nil {
|
|
+ t.Fatalf("TestDiskUsage failed: %s", err.Error())
|
|
+ }
|
|
+
|
|
used, err := resource.ParseQuantity(fmt.Sprintf("%d", total))
|
|
if err != nil {
|
|
t.Fatalf("TestDiskUsage failed: %s", err.Error())
|