diff --git a/README.md b/README.md index 173834c..0aa5eb4 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,7 @@ Options: -j, --javascript JavaScript to run on each page -o, --output Output directory name (default 'out') -p, --proxy Use proxy on given URL + -s, --skip-save-response Do not save responses in general -w, --overwrite Overwrite output files when they already exist --no-third-party Do not save responses to requests on third-party domains --third-party Only save responses to requests on third-party domains diff --git a/main.go b/main.go index c415095..7da9d8f 100644 --- a/main.go +++ b/main.go @@ -35,6 +35,7 @@ func init() { " -j, --javascript JavaScript to run on each page", " -o, --output Output directory name (default 'out')", " -p, --proxy Use proxy on given URL", + " -s, --skip-save-response Do not save responses in general", " -w, --overwrite Overwrite output files when they already exist", " --no-third-party Do not save responses to requests on third-party domains", " --third-party Only save responses to requests on third-party domains", @@ -46,15 +47,16 @@ func init() { } type options struct { - includes listArg - excludes listArg - thirdPartyOnly bool - noThirdParty bool - overwrite bool - output string - concurrency int - js string - proxy string + includes listArg + excludes listArg + thirdPartyOnly bool + noThirdParty bool + overwrite bool + skipSaveResponse bool + output string + concurrency int + js string + proxy string } func main() { @@ -67,6 +69,9 @@ func main() { flag.Var(&opts.excludes, "exclude", "") flag.Var(&opts.excludes, "e", "") + flag.BoolVar(&opts.skipSaveResponse, "skip-save-response", false, "") + flag.BoolVar(&opts.skipSaveResponse, "s", false, "") + flag.BoolVar(&opts.thirdPartyOnly, "third-party", false, "") flag.BoolVar(&opts.noThirdParty, "no-third-party", false, "") @@ -336,26 +341,27 @@ func makeListener(ctx context.Context, requestURL string, opts options) func(int err := chromedp.Run( ctx, chromedp.ActionFunc(func(ctx context.Context) error { - data, err := fetch.GetResponseBody(ev.RequestID).Do(ctx) - if err != nil { - // this function always has to return a nil error - // otherwise the ContinueRequest does not run - return nil - } - - path, err := saveResponse(ev.Request.URL, data, opts.output, opts.overwrite) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to save response data for %s: %s\n", ev.Request.URL, err) - return nil + if !opts.skipSaveResponse { + data, err := fetch.GetResponseBody(ev.RequestID).Do(ctx) + if err != nil { + // this function always has to return a nil error + // otherwise the ContinueRequest does not run + return nil + } + + path, err := saveResponse(ev.Request.URL, data, opts.output, opts.overwrite) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to save response data for %s: %s\n", ev.Request.URL, err) + return nil + } + + // save the headers etc in a separate file + err = saveMeta(path+".meta", requestURL, ev) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to save response meta data for %s: %s\n", ev.Request.URL, err) + return nil + } } - - // save the headers etc in a separate file - err = saveMeta(path+".meta", requestURL, ev) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to save response meta data for %s: %s\n", ev.Request.URL, err) - return nil - } - // Log the request fmt.Printf("%s %s %d %s\n", ev.Request.Method, ev.Request.URL, ev.ResponseStatusCode, contentType)