From 8efcc5d2149d1654558d4ecfd61840630eb49810 Mon Sep 17 00:00:00 2001 From: Fabian Heib Date: Tue, 29 Jun 2021 11:32:55 +0200 Subject: [PATCH 1/3] set http client in main instead of var threads was not set as its run be init and vars are already defined by that time meaning that the keepalive settings weren't used, resulting in creating new tcp connections --- go.mod | 8 +++++ go.sum | 104 +++++++++++++++++++++++++++++++++++++++++++++++++++++ hsbench.go | 75 +++++++++++++------------------------- 3 files changed, 136 insertions(+), 51 deletions(-) create mode 100644 go.mod create mode 100644 go.sum diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..f75c151 --- /dev/null +++ b/go.mod @@ -0,0 +1,8 @@ +module github.com/markhpc/hsbench + +go 1.16 + +require ( + code.cloudfoundry.org/bytefmt v0.0.0-20210608160410-67692ebc98de + github.com/aws/aws-sdk-go v1.38.68 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..19fcb56 --- /dev/null +++ b/go.sum @@ -0,0 +1,104 @@ +code.cloudfoundry.org/bytefmt v0.0.0-20210608160410-67692ebc98de h1:Gm/tSRP5CPFuoMUo7NDbRMbpN3h9fBNtD6bXC+Lye9I= +code.cloudfoundry.org/bytefmt v0.0.0-20210608160410-67692ebc98de/go.mod h1:YOakoAiZWNbkynB2dKOKvdxVehYGn3EH4UXq/i99hYg= +github.com/aws/aws-sdk-go v1.38.68 h1:aOG8geU4SohNp659eKBHRBgbqSrZ6jNZlfimIuJAwL8= +github.com/aws/aws-sdk-go v1.38.68/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak= +github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/hsbench.go b/hsbench.go index 385f3cf..92436c3 100644 --- a/hsbench.go +++ b/hsbench.go @@ -6,7 +6,6 @@ package main import ( "bytes" - "code.cloudfoundry.org/bytefmt" "crypto/hmac" "crypto/md5" "crypto/sha1" @@ -16,10 +15,6 @@ import ( "encoding/json" "flag" "fmt" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" "io" "io/ioutil" "log" @@ -34,6 +29,12 @@ import ( "sync" "sync/atomic" "time" + + "code.cloudfoundry.org/bytefmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" ) // Global variables @@ -47,50 +48,6 @@ var object_count_flag bool var endtime time.Time var interval float64 -// Our HTTP transport used for the roundtripper below -var HTTPTransport http.RoundTripper = &http.Transport{ - Proxy: http.ProxyFromEnvironment, - Dial: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - }).Dial, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 0, - // Set the number of idle connections to 2X the number of threads - MaxIdleConnsPerHost: 2 * threads, - MaxIdleConns: 2 * threads, - // But limit their idle time to 1 minute - IdleConnTimeout: time.Minute, - // Ignore TLS errors - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, -} - -var httpClient = &http.Client{Transport: HTTPTransport} - -func getS3Client() *s3.S3 { - // Build our config - creds := credentials.NewStaticCredentials(access_key, secret_key, "") - loglevel := aws.LogOff - // Build the rest of the configuration - awsConfig := &aws.Config{ - Region: aws.String(region), - Endpoint: aws.String(url_host), - Credentials: creds, - LogLevel: &loglevel, - S3ForcePathStyle: aws.Bool(true), - S3Disable100Continue: aws.Bool(true), - // Comment following to use default transport - HTTPClient: &http.Client{Transport: HTTPTransport}, - } - session := session.New(awsConfig) - client := s3.New(session) - if client == nil { - log.Fatalf("FATAL: Unable to create new client.") - } - // Return success - return client -} - // canonicalAmzHeaders -- return the x-amz headers canonicalized func canonicalAmzHeaders(req *http.Request) string { // Parse out all x-amz headers @@ -470,7 +427,7 @@ func (stats *Stats) finish(thread_num int) { } } -func runUpload(thread_num int, fendtime time.Time, stats *Stats) { +func runUpload(thread_num int, stats *Stats) { errcnt := 0 svc := s3.New(session.New(), cfg) for { @@ -769,7 +726,7 @@ func runWrapper(loop int, r rune) []OutputStats { log.Printf("Running Loop %d OBJECT PUT TEST", loop) stats = makeStats(loop, "PUT", threads, intervalNano) for n := 0; n < threads; n++ { - go runUpload(n, endtime, &stats) + go runUpload(n, &stats) } case 'l': log.Printf("Running Loop %d BUCKET LIST TEST", loop) @@ -931,6 +888,22 @@ func main() { // DisableParamValidation: aws.Bool(true), DisableComputeChecksums: aws.Bool(true), S3ForcePathStyle: aws.Bool(true), + HTTPClient: &http.Client{Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: time.Second, + // Set the number of idle connections to 2X the number of threads + MaxIdleConnsPerHost: 2 * threads, + MaxIdleConns: 2 * threads, + // But limit their idle time to 1 minute + IdleConnTimeout: time.Minute, + // Ignore TLS errors + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }}, } // Echo the parameters From 09780ad3e933a32184bf79cd3553fc2fb8641595 Mon Sep 17 00:00:00 2001 From: Artem Kolesnichenko Date: Mon, 10 Jul 2023 09:43:40 +0300 Subject: [PATCH 2/3] Added -single_bucket key --- README.md | 6 ++++-- hsbench.go | 10 ++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index fee5901..b0657e6 100644 --- a/README.md +++ b/README.md @@ -36,14 +36,14 @@ To leverage this tool, the following prerequisites apply: The easiest way to install hsbench is to use go's built in github support: ``` -go get github.com/markhpc/hsbench +go get github.com/temdon/hsbench_fork ``` Then in the hsbench src directory run 'go build': ``` $ pwd -/home/perf/go/src/github.com/markhpc/hsbench +/home/perf/go/src/github.com/temdon/hsbench_fork $ go build ``` @@ -83,6 +83,8 @@ OPTIONS: Number of seconds between report intervals (default 1) -s string Secret key + -single_bucket + Whether to use single bucket. If true - b (bucket count) parameter is ignored -t int Number of threads to run (default 1) -u string diff --git a/hsbench.go b/hsbench.go index 385f3cf..ed4641f 100644 --- a/hsbench.go +++ b/hsbench.go @@ -46,6 +46,7 @@ var max_keys, running_threads, bucket_count, object_count, object_size, op_count var object_count_flag bool var endtime time.Time var interval float64 +var single_bucket bool // Our HTTP transport used for the roundtripper below var HTTPTransport http.RoundTripper = &http.Transport{ @@ -834,6 +835,7 @@ func init() { myflag.Int64Var(&max_keys, "mk", 1000, "Maximum number of keys to retreive at once for bucket listings") myflag.Int64Var(&object_count, "n", -1, "Maximum number of objects <-1 for unlimited>") myflag.Int64Var(&bucket_count, "b", 1, "Number of buckets to distribute IOs across") + myflag.BoolVar(&single_bucket, "single_bucket", false, "Whether to use single bucket. If true - b (bucket count) parameter is ignored") myflag.IntVar(&duration_secs, "d", 60, "Maximum test duration in seconds <-1 for unlimited>") myflag.IntVar(&threads, "t", 1, "Number of threads to run") myflag.IntVar(&loops, "l", 1, "Number of times to repeat test") @@ -955,8 +957,12 @@ func main() { initData() // Setup the slice of buckets - for i := int64(0); i < bucket_count; i++ { - buckets = append(buckets, fmt.Sprintf("%s%012d", bucket_prefix, i)) + if single_bucket { + buckets = append(buckets, bucket_prefix) + } else { + for i := int64(0); i < bucket_count; i++ { + buckets = append(buckets, fmt.Sprintf("%s%012d", bucket_prefix, i)) + } } // Loop running the tests From 05db16dbd6c3f86ba0ce462df42b4578d245b1e4 Mon Sep 17 00:00:00 2001 From: "Alexey.Vasilyev" Date: Thu, 4 Jul 2024 14:50:27 -0400 Subject: [PATCH 3/3] multiple prefixes to distribute IO across --- hsbench.go | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/hsbench.go b/hsbench.go index e339f46..329723f 100644 --- a/hsbench.go +++ b/hsbench.go @@ -48,6 +48,8 @@ var object_count_flag bool var endtime time.Time var interval float64 var single_bucket bool +var prefix_count uint +var prefixes []string // canonicalAmzHeaders -- return the x-amz headers canonicalized func canonicalAmzHeaders(req *http.Request) string { @@ -443,7 +445,7 @@ func runUpload(thread_num int, stats *Stats) { } fileobj := bytes.NewReader(object_data) - key := fmt.Sprintf("%s%012d", object_prefix, objnum) + key := fmt.Sprintf("%s%012d", prefixes[objnum % int64(prefix_count)], objnum) r := &s3.PutObjectInput{ Bucket: &buckets[bucket_num], Key: &key, @@ -489,7 +491,7 @@ func runDownload(thread_num int, fendtime time.Time, stats *Stats) { } bucket_num := objnum % int64(bucket_count) - key := fmt.Sprintf("%s%012d", object_prefix, objnum) + key := fmt.Sprintf("%s%012d", prefixes[objnum % int64(prefix_count)], objnum) r := &s3.GetObjectInput{ Bucket: &buckets[bucket_num], Key: &key, @@ -537,7 +539,7 @@ func runDelete(thread_num int, stats *Stats) { bucket_num := objnum % int64(bucket_count) - key := fmt.Sprintf("%s%012d", object_prefix, objnum) + key := fmt.Sprintf("%s%012d", prefixes[objnum % int64(prefix_count)], objnum) r := &s3.DeleteObjectInput{ Bucket: &buckets[bucket_num], Key: &key, @@ -793,6 +795,7 @@ func init() { myflag.Int64Var(&object_count, "n", -1, "Maximum number of objects <-1 for unlimited>") myflag.Int64Var(&bucket_count, "b", 1, "Number of buckets to distribute IOs across") myflag.BoolVar(&single_bucket, "single_bucket", false, "Whether to use single bucket. If true - b (bucket count) parameter is ignored") + myflag.UintVar(&prefix_count, "prefix_count", 1, "Number of prefixes to distribute IOs across. Use only with single bucket") myflag.IntVar(&duration_secs, "d", 60, "Maximum test duration in seconds <-1 for unlimited>") myflag.IntVar(&threads, "t", 1, "Number of threads to run") myflag.IntVar(&loops, "l", 1, "Number of times to repeat test") @@ -920,6 +923,7 @@ func main() { log.Printf("max_keys=%d", max_keys) log.Printf("object_count=%d", object_count) log.Printf("bucket_count=%d", bucket_count) + log.Printf("prefix_count=%d", prefix_count) log.Printf("duration=%d", duration_secs) log.Printf("threads=%d", threads) log.Printf("loops=%d", loops) @@ -932,7 +936,19 @@ func main() { // Setup the slice of buckets if single_bucket { buckets = append(buckets, bucket_prefix) + + var base_prefix = strings.TrimRight(object_prefix, "/") + if prefix_count > 1 { + for i := uint(0); i < prefix_count; i++ { + prefixes = append(prefixes, fmt.Sprintf("%s%04d/", base_prefix, i)) + } + } else { + prefix_count = 1 + prefixes = append(prefixes, object_prefix) + } } else { + prefix_count = 1 + prefixes = append(prefixes, object_prefix) for i := int64(0); i < bucket_count; i++ { buckets = append(buckets, fmt.Sprintf("%s%012d", bucket_prefix, i)) }