From 5ab826e63a6994294b4843104e73fbca05b76d91 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 6 Feb 2023 12:29:42 +0100 Subject: [PATCH 01/17] poc: running experiments bypassing internal/engine This PoC investigates whether it would be possible to run experiments directly without using the internal/engine abstraction as the middle man. The PoC is in the context of https://github.com/ooni/ooni.org/issues/1295 --- .../dismantle/backendclient/backendclient.go | 67 ++++++++ .../dismantle/backendclient/measurement.go | 57 +++++++ .../cmd/dismantle/backendclient/submitter.go | 34 ++++ internal/cmd/dismantle/main.go | 156 ++++++++++++++++++ internal/cmd/dismantle/session.go | 81 +++++++++ .../sessionhttpclient/sessionhttpclient.go | 34 ++++ internal/model/ooapi.go | 3 + 7 files changed, 432 insertions(+) create mode 100644 internal/cmd/dismantle/backendclient/backendclient.go create mode 100644 internal/cmd/dismantle/backendclient/measurement.go create mode 100644 internal/cmd/dismantle/backendclient/submitter.go create mode 100644 internal/cmd/dismantle/main.go create mode 100644 internal/cmd/dismantle/session.go create mode 100644 internal/cmd/dismantle/sessionhttpclient/sessionhttpclient.go diff --git a/internal/cmd/dismantle/backendclient/backendclient.go b/internal/cmd/dismantle/backendclient/backendclient.go new file mode 100644 index 0000000000..096c52c37a --- /dev/null +++ b/internal/cmd/dismantle/backendclient/backendclient.go @@ -0,0 +1,67 @@ +package backendclient + +import ( + "context" + "net/url" + + "github.com/ooni/probe-cli/v3/internal/httpapi" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/ooapi" +) + +type Config struct { + KVStore model.KeyValueStore + HTTPClient model.HTTPClient + Logger model.Logger + UserAgent string + + // optional fields + BaseURL *url.URL + ProxyURL *url.URL +} + +type Client struct { + endpoint *httpapi.Endpoint +} + +func New(config *Config) *Client { + baseURL := "https://api.ooni.io/" + if config.BaseURL != nil { + baseURL = config.BaseURL.String() + } + endpoint := &httpapi.Endpoint{ + BaseURL: baseURL, + HTTPClient: config.HTTPClient, + Host: "", + Logger: config.Logger, + UserAgent: config.UserAgent, + } + backendClient := &Client{ + endpoint: endpoint, + } + return backendClient +} + +func (c *Client) CheckIn( + ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResult, error) { + return httpapi.Call(ctx, ooapi.NewDescriptorCheckIn(config), c.endpoint) +} + +func (c *Client) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { + panic("not implemented") +} + +func (c *Client) FetchTorTargets( + ctx context.Context, cc string) (result map[string]model.OOAPITorTarget, err error) { + panic("not implemented") +} + +func (c *Client) Submit(ctx context.Context, m *model.Measurement) error { + req := &model.OOAPICollectorUpdateRequest{ + Format: "json", + Content: m, + } + descriptor := newSubmitDescriptor(req, m.ReportID) + _, err := httpapi.Call(ctx, descriptor, c.endpoint) + return err +} diff --git a/internal/cmd/dismantle/backendclient/measurement.go b/internal/cmd/dismantle/backendclient/measurement.go new file mode 100644 index 0000000000..b7f392fc8e --- /dev/null +++ b/internal/cmd/dismantle/backendclient/measurement.go @@ -0,0 +1,57 @@ +package backendclient + +import ( + "fmt" + "runtime" + "time" + + "github.com/ooni/probe-cli/v3/internal/geolocate" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/platform" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/version" +) + +const dateFormat = "2006-01-02 15:04:05" + +func NewMeasurement( + location *geolocate.Results, + testName string, + testVersion string, + testStartTime time.Time, + reportID string, + softwareName string, + softwareVersion string, + input string, +) *model.Measurement { + utctimenow := time.Now().UTC() + m := &model.Measurement{ + DataFormatVersion: model.OOAPIReportDefaultDataFormatVersion, + Input: model.MeasurementTarget(input), + MeasurementStartTime: utctimenow.Format(dateFormat), + MeasurementStartTimeSaved: utctimenow, + ProbeIP: model.DefaultProbeIP, + ProbeASN: location.ASNString(), + ProbeCC: location.CountryCode, + ProbeNetworkName: location.NetworkName, + ReportID: reportID, + ResolverASN: fmt.Sprintf("AS%d", location.ResolverASN), // XXX + ResolverIP: location.ResolverIP, + ResolverNetworkName: location.ResolverNetworkName, + SoftwareName: softwareName, + SoftwareVersion: softwareVersion, + TestName: testName, + TestStartTime: testStartTime.Format(dateFormat), + TestVersion: testVersion, + } + m.AddAnnotation("architecture", runtime.GOARCH) + m.AddAnnotation("engine_name", "ooniprobe-engine") + m.AddAnnotation("engine_version", version.Version) + m.AddAnnotation("go_version", runtimex.BuildInfo.GoVersion) + m.AddAnnotation("platform", platform.Name()) + m.AddAnnotation("vcs_modified", runtimex.BuildInfo.VcsModified) + m.AddAnnotation("vcs_revision", runtimex.BuildInfo.VcsRevision) + m.AddAnnotation("vcs_time", runtimex.BuildInfo.VcsTime) + m.AddAnnotation("vcs_tool", runtimex.BuildInfo.VcsTool) + return m +} diff --git a/internal/cmd/dismantle/backendclient/submitter.go b/internal/cmd/dismantle/backendclient/submitter.go new file mode 100644 index 0000000000..8bfed04b2d --- /dev/null +++ b/internal/cmd/dismantle/backendclient/submitter.go @@ -0,0 +1,34 @@ +package backendclient + +import ( + "encoding/json" + "fmt" + "net/http" + + "github.com/ooni/probe-cli/v3/internal/httpapi" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +func newSubmitDescriptor( + req *model.OOAPICollectorUpdateRequest, reportID string) *httpapi.Descriptor[ + *model.OOAPICollectorUpdateRequest, *model.OOAPICollectorUpdateResponse] { + rawBody, err := json.Marshal(req) + runtimex.PanicOnError(err, "json.Marshal failed") + return &httpapi.Descriptor[*model.OOAPICollectorUpdateRequest, *model.OOAPICollectorUpdateResponse]{ + Accept: httpapi.ApplicationJSON, + Authorization: "", + AcceptEncodingGzip: false, + ContentType: httpapi.ApplicationJSON, + LogBody: true, + MaxBodySize: 0, + Method: http.MethodPost, + Request: &httpapi.RequestDescriptor[*model.OOAPICollectorUpdateRequest]{ + Body: rawBody, + }, + Response: &httpapi.JSONResponseDescriptor[model.OOAPICollectorUpdateResponse]{}, + Timeout: 0, + URLPath: fmt.Sprintf("/report/%s", reportID), + URLQuery: nil, + } +} diff --git a/internal/cmd/dismantle/main.go b/internal/cmd/dismantle/main.go new file mode 100644 index 0000000000..32c18b1ef7 --- /dev/null +++ b/internal/cmd/dismantle/main.go @@ -0,0 +1,156 @@ +package main + +import ( + "context" + "fmt" + "net/url" + "os" + "path/filepath" + "time" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/bytecounter" + "github.com/ooni/probe-cli/v3/internal/cmd/dismantle/backendclient" + "github.com/ooni/probe-cli/v3/internal/cmd/dismantle/sessionhttpclient" + "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivitylte" + "github.com/ooni/probe-cli/v3/internal/geolocate" + "github.com/ooni/probe-cli/v3/internal/kvstore" + "github.com/ooni/probe-cli/v3/internal/logx" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/platform" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/sessionresolver" + "github.com/ooni/probe-cli/v3/internal/tunnel" + "github.com/ooni/probe-cli/v3/internal/version" +) + +func main() { + const softwareName = "dismantle" + const softwareVersion = "0.1.0-dev" + userAgent := fmt.Sprintf( + "%s/%s ooniprobe-engine/%s", + softwareName, softwareVersion, + version.Version, + ) + + logHandler := logx.NewHandlerWithDefaultSettings() + logHandler.Emoji = true + logger := &log.Logger{Level: log.InfoLevel, Handler: logHandler} + progressBar := model.NewPrinterCallbacks(logger) + counter := bytecounter.New() + home := filepath.Join(os.Getenv("HOME"), ".miniooni") + statedir := filepath.Join(home, "kvstore2") + ctx := context.Background() + tunnelDir := filepath.Join(home, "tunnel") + runtimex.Try0(os.MkdirAll(tunnelDir, 0700)) + + kvstore := runtimex.Try1(kvstore.NewFS(statedir)) + + tunnelConfig := &tunnel.Config{ + Name: "tor", + TunnelDir: tunnelDir, + Logger: logger, + } + tunnel, _ := runtimex.Try2(tunnel.Start(ctx, tunnelConfig)) + defer tunnel.Stop() + proxyURL := tunnel.SOCKS5ProxyURL() + + sessionResolver := &sessionresolver.Resolver{ + ByteCounter: counter, + KVStore: kvstore, + Logger: logger, + ProxyURL: proxyURL, + } + defer sessionResolver.CloseIdleConnections() + + geolocateConfig := &geolocate.Config{ + Resolver: sessionResolver, + Logger: logger, + UserAgent: model.HTTPHeaderUserAgent, + } + geolocateTask := geolocate.NewTask(*geolocateConfig) // XXX + location := runtimex.Try1(geolocateTask.Run(ctx)) + logger.Infof("%+v", location) + + sessionHTTPClientConfig := &sessionhttpclient.Config{ + ByteCounter: counter, + Logger: logger, + Resolver: sessionResolver, + ProxyURL: proxyURL, + } + sessionHTTPClient := sessionhttpclient.New(sessionHTTPClientConfig) + defer sessionHTTPClient.CloseIdleConnections() + + backendClientConfig := &backendclient.Config{ + KVStore: kvstore, + HTTPClient: sessionHTTPClient, + Logger: logger, + UserAgent: userAgent, + BaseURL: nil, + ProxyURL: proxyURL, + } + backendClient := backendclient.New(backendClientConfig) + + checkInConfig := &model.OOAPICheckInConfig{ + Charging: false, + OnWiFi: false, + Platform: platform.Name(), + ProbeASN: location.ASNString(), + ProbeCC: location.CountryCode, + RunType: "manual", + SoftwareName: softwareName, + SoftwareVersion: softwareName, + WebConnectivity: model.OOAPICheckInConfigWebConnectivity{ + CategoryCodes: []string{}, + }, + } + checkInResult := runtimex.Try1(backendClient.CheckIn(ctx, checkInConfig)) + logger.Infof("%+v", checkInResult) + + runtimex.Assert(checkInResult.Tests.WebConnectivity != nil, "no web connectivity info") + reportID := checkInResult.Tests.WebConnectivity.ReportID + + experimentSession := &experimentSession{ + httpClient: sessionHTTPClient, + location: location, + logger: logger, + testHelpers: checkInResult.Conf.TestHelpers, + userAgent: userAgent, + } + + testStartTime := time.Now() + for _, input := range checkInResult.Tests.WebConnectivity.URLs { + cfg := &webconnectivitylte.Config{} + runner := webconnectivitylte.NewExperimentMeasurer(cfg) + measurement := backendclient.NewMeasurement( + location, runner.ExperimentName(), runner.ExperimentVersion(), + testStartTime, reportID, softwareName, softwareVersion, input.URL, + ) + args := &model.ExperimentArgs{ + Callbacks: progressBar, + Measurement: measurement, + Session: experimentSession, + } + if err := runner.Run(ctx, args); err != nil { + logger.Warnf("runner.Run failed: %s", err.Error()) + } + if err := backendClient.Submit(ctx, measurement); err != nil { + logger.Warnf("backendClient.Submit failed: %s", err.Error()) + } + log.Infof("measurement URL: %s", makeExplorerURL(reportID, input.URL)) + } +} + +func makeExplorerURL(reportID, input string) string { + query := url.Values{} + query.Add("input", input) + explorerURL := &url.URL{ + Scheme: "https", + Host: "explorer.ooni.org", + Path: fmt.Sprintf("/measurement/%s", reportID), + RawQuery: query.Encode(), + Fragment: "", + RawFragment: "", + } + return explorerURL.String() +} diff --git a/internal/cmd/dismantle/session.go b/internal/cmd/dismantle/session.go new file mode 100644 index 0000000000..1f2131eb57 --- /dev/null +++ b/internal/cmd/dismantle/session.go @@ -0,0 +1,81 @@ +package main + +import ( + "context" + + "github.com/ooni/probe-cli/v3/internal/geolocate" + "github.com/ooni/probe-cli/v3/internal/model" +) + +type experimentSession struct { + httpClient model.HTTPClient + location *geolocate.Results + logger model.Logger + testHelpers map[string][]model.OOAPIService + userAgent string +} + +var _ model.ExperimentSession = &experimentSession{} + +// DefaultHTTPClient implements model.ExperimentSession +func (es *experimentSession) DefaultHTTPClient() model.HTTPClient { + return es.httpClient +} + +// FetchPsiphonConfig implements model.ExperimentSession +func (es *experimentSession) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { + // FIXME: we need to call the backend API for this I think? + panic("unimplemented") +} + +// FetchTorTargets implements model.ExperimentSession +func (es *experimentSession) FetchTorTargets(ctx context.Context, cc string) (map[string]model.OOAPITorTarget, error) { + // FIXME: we need to call the backend API for this I think? + panic("unimplemented") +} + +// GetTestHelpersByName implements model.ExperimentSession +func (es *experimentSession) GetTestHelpersByName(name string) ([]model.OOAPIService, bool) { + value, found := es.testHelpers[name] + return value, found +} + +// Logger implements model.ExperimentSession +func (es *experimentSession) Logger() model.Logger { + return es.logger +} + +// ProbeCC implements model.ExperimentSession +func (es *experimentSession) ProbeCC() string { + return es.location.CountryCode +} + +// ResolverIP implements model.ExperimentSession +func (es *experimentSession) ResolverIP() string { + return es.location.ResolverIP +} + +// TempDir implements model.ExperimentSession +func (es *experimentSession) TempDir() string { + panic("unimplemented") // FIXME +} + +// TorArgs implements model.ExperimentSession +func (es *experimentSession) TorArgs() []string { + panic("unimplemented") // FIXME +} + +// TorBinary implements model.ExperimentSession +func (es *experimentSession) TorBinary() string { + panic("unimplemented") // FIXME +} + +// TunnelDir implements model.ExperimentSession +func (es *experimentSession) TunnelDir() string { + panic("unimplemented") // FIXME +} + +// UserAgent implements model.ExperimentSession +func (es *experimentSession) UserAgent() string { + return es.userAgent +} diff --git a/internal/cmd/dismantle/sessionhttpclient/sessionhttpclient.go b/internal/cmd/dismantle/sessionhttpclient/sessionhttpclient.go new file mode 100644 index 0000000000..c7b5598170 --- /dev/null +++ b/internal/cmd/dismantle/sessionhttpclient/sessionhttpclient.go @@ -0,0 +1,34 @@ +// Package sessionhttpclient creates an HTTP client for +// a measurement session. We will use this client for +// communicating with the OONI backend. +package sessionhttpclient + +import ( + "net/url" + + "github.com/ooni/probe-cli/v3/internal/bytecounter" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// Config contains config for creating a new session HTTP client. +type Config struct { + ByteCounter *bytecounter.Counter + Logger model.Logger + Resolver model.Resolver + + // optional fields + ProxyURL *url.URL +} + +// New creates a new HTTP client to be used during a measurement +// session to communicate with the OONI backend. +func New(config *Config) model.HTTPClient { + dialer := netxlite.NewDialerWithResolver(config.Logger, config.Resolver) + dialer = netxlite.MaybeWrapWithProxyDialer(dialer, config.ProxyURL) + handshaker := netxlite.NewTLSHandshakerStdlib(config.Logger) + tlsDialer := netxlite.NewTLSDialer(dialer, handshaker) + txp := netxlite.NewHTTPTransport(config.Logger, dialer, tlsDialer) + txp = bytecounter.MaybeWrapHTTPTransport(txp, config.ByteCounter) + return netxlite.NewHTTPClient(txp) +} diff --git a/internal/model/ooapi.go b/internal/model/ooapi.go index 925b112741..dd2ca3f8c8 100644 --- a/internal/model/ooapi.go +++ b/internal/model/ooapi.go @@ -87,6 +87,9 @@ type OOAPICheckInResult struct { type OOAPICheckInResultConfig struct { // Features contains feature flags. Features map[string]bool `json:"features"` + + // TestHelpers contains test-helpers information. + TestHelpers map[string][]OOAPIService `json:"test_helpers"` } // OOAPICheckReportIDResponse is the check-report-id API response. From f25f14a9e5b305c5faf93635ad451186577073f2 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 8 Feb 2023 06:58:54 +0000 Subject: [PATCH 02/17] refactor: create the sessionhttpclient package --- .../cmd/dismantle/backendclient/measurement.go | 2 ++ internal/cmd/dismantle/main.go | 2 +- .../sessionhttpclient/sessionhttpclient.go | 14 ++++++++++---- 3 files changed, 13 insertions(+), 5 deletions(-) rename internal/{cmd/dismantle => }/sessionhttpclient/sessionhttpclient.go (74%) diff --git a/internal/cmd/dismantle/backendclient/measurement.go b/internal/cmd/dismantle/backendclient/measurement.go index b7f392fc8e..5b54f695ff 100644 --- a/internal/cmd/dismantle/backendclient/measurement.go +++ b/internal/cmd/dismantle/backendclient/measurement.go @@ -12,8 +12,10 @@ import ( "github.com/ooni/probe-cli/v3/internal/version" ) +// dateFormat is the data format used to fill a measurement. const dateFormat = "2006-01-02 15:04:05" +// NewMeasurement constructs a new measurement. func NewMeasurement( location *geolocate.Results, testName string, diff --git a/internal/cmd/dismantle/main.go b/internal/cmd/dismantle/main.go index 32c18b1ef7..842516161d 100644 --- a/internal/cmd/dismantle/main.go +++ b/internal/cmd/dismantle/main.go @@ -11,7 +11,6 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/bytecounter" "github.com/ooni/probe-cli/v3/internal/cmd/dismantle/backendclient" - "github.com/ooni/probe-cli/v3/internal/cmd/dismantle/sessionhttpclient" "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivitylte" "github.com/ooni/probe-cli/v3/internal/geolocate" "github.com/ooni/probe-cli/v3/internal/kvstore" @@ -19,6 +18,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/platform" "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/sessionhttpclient" "github.com/ooni/probe-cli/v3/internal/sessionresolver" "github.com/ooni/probe-cli/v3/internal/tunnel" "github.com/ooni/probe-cli/v3/internal/version" diff --git a/internal/cmd/dismantle/sessionhttpclient/sessionhttpclient.go b/internal/sessionhttpclient/sessionhttpclient.go similarity index 74% rename from internal/cmd/dismantle/sessionhttpclient/sessionhttpclient.go rename to internal/sessionhttpclient/sessionhttpclient.go index c7b5598170..b144b7187f 100644 --- a/internal/cmd/dismantle/sessionhttpclient/sessionhttpclient.go +++ b/internal/sessionhttpclient/sessionhttpclient.go @@ -13,15 +13,21 @@ import ( // Config contains config for creating a new session HTTP client. type Config struct { + // ByteCounter is the MANDATORY byte counter to use. ByteCounter *bytecounter.Counter - Logger model.Logger - Resolver model.Resolver - // optional fields + // Logger is the MANDATORY logger to use. + Logger model.Logger + + // ProxyURL is the OPTIONAL proxy URL that the HTTPClient + // returned by New should be using. ProxyURL *url.URL + + // Resolver is the MANDATORY resolver to use. + Resolver model.Resolver } -// New creates a new HTTP client to be used during a measurement +// New creates a new HTTPClient to be used during a measurement // session to communicate with the OONI backend. func New(config *Config) model.HTTPClient { dialer := netxlite.NewDialerWithResolver(config.Logger, config.Resolver) From 258b8e09c9a5f2a961da3efc0ed18d8be15aad89 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 8 Feb 2023 07:01:00 +0000 Subject: [PATCH 03/17] doc(model): document location.go data structure --- internal/model/location.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/internal/model/location.go b/internal/model/location.go index 36322f8abc..5a35a1a85e 100644 --- a/internal/model/location.go +++ b/internal/model/location.go @@ -1,12 +1,22 @@ package model -// LocationProvider is an interface that returns the current location. The -// [engine.Session] struct implements this interface. +// LocationProvider is an interface that returns the current location. type LocationProvider interface { + // ProbeASN is the ASN associated with ProbeIP. ProbeASN() uint + + // ProbeASNString returns the probe ASN as the AS%d string. ProbeASNString() string + + // ProbeCC is the country code associated with ProbeIP. ProbeCC() string + + // ProbeIP is the probe IP address. ProbeIP() string + + // ProbeNetworkName is the name of the ProbeASN. ProbeNetworkName() string + + // ResolverIP is the IP of the resolver. ResolverIP() string } From f1d7aedb979bd25c55f5aa36faf50eeeca7ccd15 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 8 Feb 2023 07:04:58 +0000 Subject: [PATCH 04/17] refactor: facilitate making geolocate.Results a Location --- .../dismantle/backendclient/measurement.go | 8 +-- internal/cmd/dismantle/main.go | 2 +- internal/cmd/dismantle/session.go | 2 +- internal/engine/experiment_test.go | 8 +-- internal/engine/session.go | 8 +-- internal/geolocate/geolocate.go | 48 +++++++-------- internal/geolocate/geolocate_test.go | 58 +++++++++---------- pkg/oonimkall/session.go | 6 +- 8 files changed, 70 insertions(+), 70 deletions(-) diff --git a/internal/cmd/dismantle/backendclient/measurement.go b/internal/cmd/dismantle/backendclient/measurement.go index 5b54f695ff..8ccf578202 100644 --- a/internal/cmd/dismantle/backendclient/measurement.go +++ b/internal/cmd/dismantle/backendclient/measurement.go @@ -33,13 +33,13 @@ func NewMeasurement( MeasurementStartTime: utctimenow.Format(dateFormat), MeasurementStartTimeSaved: utctimenow, ProbeIP: model.DefaultProbeIP, - ProbeASN: location.ASNString(), + ProbeASN: location.ProbeASNString(), ProbeCC: location.CountryCode, ProbeNetworkName: location.NetworkName, ReportID: reportID, - ResolverASN: fmt.Sprintf("AS%d", location.ResolverASN), // XXX - ResolverIP: location.ResolverIP, - ResolverNetworkName: location.ResolverNetworkName, + ResolverASN: fmt.Sprintf("AS%d", location.ResolverASNumber), // XXX + ResolverIP: location.ResolverIPAddr, + ResolverNetworkName: location.ResolverASNetworkName, SoftwareName: softwareName, SoftwareVersion: softwareVersion, TestName: testName, diff --git a/internal/cmd/dismantle/main.go b/internal/cmd/dismantle/main.go index 842516161d..6d7b248b71 100644 --- a/internal/cmd/dismantle/main.go +++ b/internal/cmd/dismantle/main.go @@ -95,7 +95,7 @@ func main() { Charging: false, OnWiFi: false, Platform: platform.Name(), - ProbeASN: location.ASNString(), + ProbeASN: location.ProbeASNString(), ProbeCC: location.CountryCode, RunType: "manual", SoftwareName: softwareName, diff --git a/internal/cmd/dismantle/session.go b/internal/cmd/dismantle/session.go index 1f2131eb57..3806e508ba 100644 --- a/internal/cmd/dismantle/session.go +++ b/internal/cmd/dismantle/session.go @@ -52,7 +52,7 @@ func (es *experimentSession) ProbeCC() string { // ResolverIP implements model.ExperimentSession func (es *experimentSession) ResolverIP() string { - return es.location.ResolverIP + return es.location.ResolverIPAddr } // TempDir implements model.ExperimentSession diff --git a/internal/engine/experiment_test.go b/internal/engine/experiment_test.go index 84298039fb..740b453335 100644 --- a/internal/engine/experiment_test.go +++ b/internal/engine/experiment_test.go @@ -24,7 +24,7 @@ func TestExperimentHonoursSharingDefaults(t *testing.T) { } allspecs := []spec{{ name: "probeIP", - locationInfo: &geolocate.Results{ProbeIP: "8.8.8.8"}, + locationInfo: &geolocate.Results{IPAddr: "8.8.8.8"}, expect: func(m *model.Measurement) bool { return m.ProbeIP == model.DefaultProbeIP }, @@ -48,19 +48,19 @@ func TestExperimentHonoursSharingDefaults(t *testing.T) { }, }, { name: "resolverIP", - locationInfo: &geolocate.Results{ResolverIP: "9.9.9.9"}, + locationInfo: &geolocate.Results{ResolverIPAddr: "9.9.9.9"}, expect: func(m *model.Measurement) bool { return m.ResolverIP == "9.9.9.9" }, }, { name: "resolverASN", - locationInfo: &geolocate.Results{ResolverASN: 44}, + locationInfo: &geolocate.Results{ResolverASNumber: 44}, expect: func(m *model.Measurement) bool { return m.ResolverASN == "AS44" }, }, { name: "resolverNetworkName", - locationInfo: &geolocate.Results{ResolverNetworkName: "Google LLC"}, + locationInfo: &geolocate.Results{ResolverASNetworkName: "Google LLC"}, expect: func(m *model.Measurement) bool { return m.ResolverNetworkName == "Google LLC" }, diff --git a/internal/engine/session.go b/internal/engine/session.go index 3169d711ca..010abededa 100644 --- a/internal/engine/session.go +++ b/internal/engine/session.go @@ -523,7 +523,7 @@ func (s *Session) ProbeIP() string { s.mu.Lock() ip := model.DefaultProbeIP if s.location != nil { - ip = s.location.ProbeIP + ip = s.location.IPAddr } return ip } @@ -544,7 +544,7 @@ func (s *Session) ResolverASN() uint { s.mu.Lock() asn := model.DefaultResolverASN if s.location != nil { - asn = s.location.ResolverASN + asn = s.location.ResolverASNumber } return asn } @@ -555,7 +555,7 @@ func (s *Session) ResolverIP() string { s.mu.Lock() ip := model.DefaultResolverIP if s.location != nil { - ip = s.location.ResolverIP + ip = s.location.ResolverIPAddr } return ip } @@ -566,7 +566,7 @@ func (s *Session) ResolverNetworkName() string { s.mu.Lock() nn := model.DefaultResolverNetworkName if s.location != nil { - nn = s.location.ResolverNetworkName + nn = s.location.ResolverASNetworkName } return nn } diff --git a/internal/geolocate/geolocate.go b/internal/geolocate/geolocate.go index a9c6f5f23d..618fd005ce 100644 --- a/internal/geolocate/geolocate.go +++ b/internal/geolocate/geolocate.go @@ -24,21 +24,21 @@ type Results struct { // NetworkName is the network name. NetworkName string - // IP is the probe IP. - ProbeIP string + // IPAddr is the probe IPAddr. + IPAddr string - // ResolverASN is the resolver ASN. - ResolverASN uint + // ResolverASNumber is the resolver ASN. + ResolverASNumber uint - // ResolverIP is the resolver IP. - ResolverIP string + // ResolverIPAddr is the resolver IP. + ResolverIPAddr string - // ResolverNetworkName is the resolver network name. - ResolverNetworkName string + // ResolverASNetworkName is the resolver network name. + ResolverASNetworkName string } -// ASNString returns the ASN as a string. -func (r *Results) ASNString() string { +// ProbeASNString returns the ASN as a string. +func (r *Results) ProbeASNString() string { return fmt.Sprintf("AS%d", r.ASN) } @@ -110,26 +110,26 @@ type Task struct { func (op Task) Run(ctx context.Context) (*Results, error) { var err error out := &Results{ - ASN: model.DefaultProbeASN, - CountryCode: model.DefaultProbeCC, - NetworkName: model.DefaultProbeNetworkName, - ProbeIP: model.DefaultProbeIP, - ResolverASN: model.DefaultResolverASN, - ResolverIP: model.DefaultResolverIP, - ResolverNetworkName: model.DefaultResolverNetworkName, + ASN: model.DefaultProbeASN, + CountryCode: model.DefaultProbeCC, + NetworkName: model.DefaultProbeNetworkName, + IPAddr: model.DefaultProbeIP, + ResolverASNumber: model.DefaultResolverASN, + ResolverIPAddr: model.DefaultResolverIP, + ResolverASNetworkName: model.DefaultResolverNetworkName, } ip, err := op.probeIPLookupper.LookupProbeIP(ctx) if err != nil { return out, fmt.Errorf("lookupProbeIP failed: %w", err) } - out.ProbeIP = ip - asn, networkName, err := op.probeASNLookupper.LookupASN(out.ProbeIP) + out.IPAddr = ip + asn, networkName, err := op.probeASNLookupper.LookupASN(out.IPAddr) if err != nil { return out, fmt.Errorf("lookupASN failed: %w", err) } out.ASN = asn out.NetworkName = networkName - cc, err := op.countryLookupper.LookupCC(out.ProbeIP) + cc, err := op.countryLookupper.LookupCC(out.IPAddr) if err != nil { return out, fmt.Errorf("lookupProbeCC failed: %w", err) } @@ -143,14 +143,14 @@ func (op Task) Run(ctx context.Context) (*Results, error) { if err != nil { return out, nil // intentional } - out.ResolverIP = resolverIP + out.ResolverIPAddr = resolverIP resolverASN, resolverNetworkName, err := op.resolverASNLookupper.LookupASN( - out.ResolverIP, + out.ResolverIPAddr, ) if err != nil { return out, nil // intentional } - out.ResolverASN = resolverASN - out.ResolverNetworkName = resolverNetworkName + out.ResolverASNumber = resolverASN + out.ResolverASNetworkName = resolverNetworkName return out, nil } diff --git a/internal/geolocate/geolocate_test.go b/internal/geolocate/geolocate_test.go index 9d10917531..004d83fc44 100644 --- a/internal/geolocate/geolocate_test.go +++ b/internal/geolocate/geolocate_test.go @@ -36,16 +36,16 @@ func TestLocationLookupCannotLookupProbeIP(t *testing.T) { if out.NetworkName != model.DefaultProbeNetworkName { t.Fatal("invalid NetworkName value") } - if out.ProbeIP != model.DefaultProbeIP { + if out.IPAddr != model.DefaultProbeIP { t.Fatal("invalid ProbeIP value") } - if out.ResolverASN != model.DefaultResolverASN { + if out.ResolverASNumber != model.DefaultResolverASN { t.Fatal("invalid ResolverASN value") } - if out.ResolverIP != model.DefaultResolverIP { + if out.ResolverIPAddr != model.DefaultResolverIP { t.Fatal("invalid ResolverIP value") } - if out.ResolverNetworkName != model.DefaultResolverNetworkName { + if out.ResolverASNetworkName != model.DefaultResolverNetworkName { t.Fatal("invalid ResolverNetworkName value") } } @@ -80,16 +80,16 @@ func TestLocationLookupCannotLookupProbeASN(t *testing.T) { if out.NetworkName != model.DefaultProbeNetworkName { t.Fatal("invalid NetworkName value") } - if out.ProbeIP != "1.2.3.4" { + if out.IPAddr != "1.2.3.4" { t.Fatal("invalid ProbeIP value") } - if out.ResolverASN != model.DefaultResolverASN { + if out.ResolverASNumber != model.DefaultResolverASN { t.Fatal("invalid ResolverASN value") } - if out.ResolverIP != model.DefaultResolverIP { + if out.ResolverIPAddr != model.DefaultResolverIP { t.Fatal("invalid ResolverIP value") } - if out.ResolverNetworkName != model.DefaultResolverNetworkName { + if out.ResolverASNetworkName != model.DefaultResolverNetworkName { t.Fatal("invalid ResolverNetworkName value") } } @@ -124,16 +124,16 @@ func TestLocationLookupCannotLookupProbeCC(t *testing.T) { if out.NetworkName != "1234.com" { t.Fatal("invalid NetworkName value") } - if out.ProbeIP != "1.2.3.4" { + if out.IPAddr != "1.2.3.4" { t.Fatal("invalid ProbeIP value") } - if out.ResolverASN != model.DefaultResolverASN { + if out.ResolverASNumber != model.DefaultResolverASN { t.Fatal("invalid ResolverASN value") } - if out.ResolverIP != model.DefaultResolverIP { + if out.ResolverIPAddr != model.DefaultResolverIP { t.Fatal("invalid ResolverIP value") } - if out.ResolverNetworkName != model.DefaultResolverNetworkName { + if out.ResolverASNetworkName != model.DefaultResolverNetworkName { t.Fatal("invalid ResolverNetworkName value") } } @@ -169,19 +169,19 @@ func TestLocationLookupCannotLookupResolverIP(t *testing.T) { if out.NetworkName != "1234.com" { t.Fatal("invalid NetworkName value") } - if out.ProbeIP != "1.2.3.4" { + if out.IPAddr != "1.2.3.4" { t.Fatal("invalid ProbeIP value") } if out.didResolverLookup != true { t.Fatal("invalid DidResolverLookup value") } - if out.ResolverASN != model.DefaultResolverASN { + if out.ResolverASNumber != model.DefaultResolverASN { t.Fatal("invalid ResolverASN value") } - if out.ResolverIP != model.DefaultResolverIP { + if out.ResolverIPAddr != model.DefaultResolverIP { t.Fatal("invalid ResolverIP value") } - if out.ResolverNetworkName != model.DefaultResolverNetworkName { + if out.ResolverASNetworkName != model.DefaultResolverNetworkName { t.Fatal("invalid ResolverNetworkName value") } } @@ -209,19 +209,19 @@ func TestLocationLookupCannotLookupResolverNetworkName(t *testing.T) { if out.NetworkName != "1234.com" { t.Fatal("invalid NetworkName value") } - if out.ProbeIP != "1.2.3.4" { + if out.IPAddr != "1.2.3.4" { t.Fatal("invalid ProbeIP value") } if out.didResolverLookup != true { t.Fatal("invalid DidResolverLookup value") } - if out.ResolverASN != model.DefaultResolverASN { - t.Fatalf("invalid ResolverASN value: %+v", out.ResolverASN) + if out.ResolverASNumber != model.DefaultResolverASN { + t.Fatalf("invalid ResolverASN value: %+v", out.ResolverASNumber) } - if out.ResolverIP != "4.3.2.1" { - t.Fatalf("invalid ResolverIP value: %+v", out.ResolverIP) + if out.ResolverIPAddr != "4.3.2.1" { + t.Fatalf("invalid ResolverIP value: %+v", out.ResolverIPAddr) } - if out.ResolverNetworkName != model.DefaultResolverNetworkName { + if out.ResolverASNetworkName != model.DefaultResolverNetworkName { t.Fatal("invalid ResolverNetworkName value") } } @@ -248,19 +248,19 @@ func TestLocationLookupSuccessWithResolverLookup(t *testing.T) { if out.NetworkName != "1234.com" { t.Fatal("invalid NetworkName value") } - if out.ProbeIP != "1.2.3.4" { + if out.IPAddr != "1.2.3.4" { t.Fatal("invalid ProbeIP value") } if out.didResolverLookup != true { t.Fatal("invalid DidResolverLookup value") } - if out.ResolverASN != 4321 { - t.Fatalf("invalid ResolverASN value: %+v", out.ResolverASN) + if out.ResolverASNumber != 4321 { + t.Fatalf("invalid ResolverASN value: %+v", out.ResolverASNumber) } - if out.ResolverIP != "4.3.2.1" { - t.Fatalf("invalid ResolverIP value: %+v", out.ResolverIP) + if out.ResolverIPAddr != "4.3.2.1" { + t.Fatalf("invalid ResolverIP value: %+v", out.ResolverIPAddr) } - if out.ResolverNetworkName != "4321.com" { + if out.ResolverASNetworkName != "4321.com" { t.Fatal("invalid ResolverNetworkName value") } } @@ -281,7 +281,7 @@ func TestSmoke(t *testing.T) { func TestASNStringWorks(t *testing.T) { r := Results{ASN: 1234} - if r.ASNString() != "AS1234" { + if r.ProbeASNString() != "AS1234" { t.Fatal("unexpected result") } } diff --git a/pkg/oonimkall/session.go b/pkg/oonimkall/session.go index 5ec01dc34d..b717a04aac 100644 --- a/pkg/oonimkall/session.go +++ b/pkg/oonimkall/session.go @@ -288,9 +288,9 @@ func (sess *Session) Geolocate(ctx *Context) (*GeolocateResults, error) { return nil, err } return &GeolocateResults{ - ASN: info.ASNString(), + ASN: info.ProbeASNString(), Country: info.CountryCode, - IP: info.ProbeIP, + IP: info.IPAddr, Org: info.NetworkName, }, nil } @@ -470,7 +470,7 @@ func (sess *Session) CheckIn(ctx *Context, config *CheckInConfig) (*CheckInInfo, Charging: config.Charging, OnWiFi: config.OnWiFi, Platform: config.Platform, - ProbeASN: info.ASNString(), + ProbeASN: info.ProbeASNString(), ProbeCC: info.CountryCode, RunType: model.RunType(config.RunType), SoftwareVersion: config.SoftwareVersion, From c1aa819d685fce8ac332b423aa924565a3308bc9 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 8 Feb 2023 07:09:25 +0000 Subject: [PATCH 05/17] fix(model): make locationProvider model complete --- internal/database/actions_test.go | 17 +++++++++++++++++ internal/model/location.go | 9 +++++++++ internal/model/mocks/location.go | 30 ++++++++++++++++++++++++------ 3 files changed, 50 insertions(+), 6 deletions(-) diff --git a/internal/database/actions_test.go b/internal/database/actions_test.go index 390ecf6d25..bbadfecc98 100644 --- a/internal/database/actions_test.go +++ b/internal/database/actions_test.go @@ -20,6 +20,8 @@ type locationInfo struct { resolverIP string } +var _ model.LocationProvider = &locationInfo{} + func (lp *locationInfo) ProbeASN() uint { return lp.asn } @@ -44,6 +46,21 @@ func (lp *locationInfo) ResolverIP() string { return lp.resolverIP } +// ResolverASN implements model.LocationProvider +func (*locationInfo) ResolverASN() uint { + panic("unimplemented") +} + +// ResolverASNString implements model.LocationProvider +func (*locationInfo) ResolverASNString() string { + panic("unimplemented") +} + +// ResolverNetworkName implements model.LocationProvider +func (*locationInfo) ResolverNetworkName() string { + panic("unimplemented") +} + func TestNewDatabase(t *testing.T) { t.Run("with empty path", func(t *testing.T) { dbpath := "" diff --git a/internal/model/location.go b/internal/model/location.go index 5a35a1a85e..fad7e0cb6d 100644 --- a/internal/model/location.go +++ b/internal/model/location.go @@ -19,4 +19,13 @@ type LocationProvider interface { // ResolverIP is the IP of the resolver. ResolverIP() string + + // ResolverASN is the resolver ASN. + ResolverASN() uint + + // ResolverASNString is the resolver ASN as the AS%d string. + ResolverASNString() string + + // ResolverNetworkName is the name of the ResolverASN. + ResolverNetworkName() string } diff --git a/internal/model/mocks/location.go b/internal/model/mocks/location.go index 2c0ecc7e6a..a48b71acf7 100644 --- a/internal/model/mocks/location.go +++ b/internal/model/mocks/location.go @@ -3,12 +3,15 @@ package mocks import "github.com/ooni/probe-cli/v3/internal/model" type LocationProvider struct { - MockProbeASN func() uint - MockProbeASNString func() string - MockProbeCC func() string - MockProbeIP func() string - MockProbeNetworkName func() string - MockResolverIP func() string + MockProbeASN func() uint + MockProbeASNString func() string + MockProbeCC func() string + MockProbeIP func() string + MockProbeNetworkName func() string + MockResolverIP func() string + MockResolverASN func() uint + MockResolverASNString func() string + MockResolverNetworkName func() string } var _ model.LocationProvider = &LocationProvider{} @@ -42,3 +45,18 @@ func (loc *LocationProvider) ProbeNetworkName() string { func (loc *LocationProvider) ResolverIP() string { return loc.MockResolverIP() } + +// ResolverASN implements model.LocationProvider +func (loc *LocationProvider) ResolverASN() uint { + return loc.MockResolverASN() +} + +// ResolverASNString implements model.LocationProvider +func (loc *LocationProvider) ResolverASNString() string { + return loc.MockResolverASNString() +} + +// ResolverNetworkName implements model.LocationProvider +func (loc *LocationProvider) ResolverNetworkName() string { + return loc.MockResolverNetworkName() +} From 2a57df7ba075b8dcdb267885dcab7fe18d541a06 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 8 Feb 2023 07:13:39 +0000 Subject: [PATCH 06/17] refactor: rewriten measurement constructor to use locationProvider --- .../dismantle/backendclient/measurement.go | 14 +++---- internal/geolocate/geolocate.go | 42 +++++++++++++++++++ 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/internal/cmd/dismantle/backendclient/measurement.go b/internal/cmd/dismantle/backendclient/measurement.go index 8ccf578202..eaacbe0db0 100644 --- a/internal/cmd/dismantle/backendclient/measurement.go +++ b/internal/cmd/dismantle/backendclient/measurement.go @@ -1,11 +1,9 @@ package backendclient import ( - "fmt" "runtime" "time" - "github.com/ooni/probe-cli/v3/internal/geolocate" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/platform" "github.com/ooni/probe-cli/v3/internal/runtimex" @@ -17,7 +15,7 @@ const dateFormat = "2006-01-02 15:04:05" // NewMeasurement constructs a new measurement. func NewMeasurement( - location *geolocate.Results, + location model.LocationProvider, testName string, testVersion string, testStartTime time.Time, @@ -34,12 +32,12 @@ func NewMeasurement( MeasurementStartTimeSaved: utctimenow, ProbeIP: model.DefaultProbeIP, ProbeASN: location.ProbeASNString(), - ProbeCC: location.CountryCode, - ProbeNetworkName: location.NetworkName, + ProbeCC: location.ProbeCC(), + ProbeNetworkName: location.ProbeNetworkName(), ReportID: reportID, - ResolverASN: fmt.Sprintf("AS%d", location.ResolverASNumber), // XXX - ResolverIP: location.ResolverIPAddr, - ResolverNetworkName: location.ResolverASNetworkName, + ResolverASN: location.ResolverASNString(), + ResolverIP: location.ResolverIP(), + ResolverNetworkName: location.ResolverNetworkName(), SoftwareName: softwareName, SoftwareVersion: softwareVersion, TestName: testName, diff --git a/internal/geolocate/geolocate.go b/internal/geolocate/geolocate.go index 618fd005ce..6805a9ff7d 100644 --- a/internal/geolocate/geolocate.go +++ b/internal/geolocate/geolocate.go @@ -37,11 +37,53 @@ type Results struct { ResolverASNetworkName string } +var _ model.LocationProvider = &Results{} + +// ProbeASN implements model.LocationProvider +func (r *Results) ProbeASN() uint { + return r.ASN +} + // ProbeASNString returns the ASN as a string. func (r *Results) ProbeASNString() string { return fmt.Sprintf("AS%d", r.ASN) } +// ProbeCC implements model.LocationProvider +func (r *Results) ProbeCC() string { + return r.CountryCode +} + +// ProbeIP implements model.LocationProvider +func (r *Results) ProbeIP() string { + return r.IPAddr +} + +// ProbeNetworkName implements model.LocationProvider +func (r *Results) ProbeNetworkName() string { + return r.NetworkName +} + +// ResolverASN implements model.LocationProvider +func (r *Results) ResolverASN() uint { + return r.ResolverASNumber +} + +// ResolverASNString implements model.LocationProvider +func (r *Results) ResolverASNString() string { + return fmt.Sprintf("AS%d", r.ResolverASNumber) +} + +// ResolverIP implements model.LocationProvider +func (r *Results) ResolverIP() string { + return r.ResolverIPAddr +} + +// ResolverNetworkName implements model.LocationProvider +func (r *Results) ResolverNetworkName() string { + return r.ResolverASNetworkName +} + type probeIPLookupper interface { LookupProbeIP(ctx context.Context) (addr string, err error) } From f2572e6161aa8213b534165bf901134a32056421 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 8 Feb 2023 07:15:44 +0000 Subject: [PATCH 07/17] refactor: avoid & and * dance --- internal/cmd/dismantle/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/cmd/dismantle/main.go b/internal/cmd/dismantle/main.go index 6d7b248b71..8f2dc99012 100644 --- a/internal/cmd/dismantle/main.go +++ b/internal/cmd/dismantle/main.go @@ -63,12 +63,12 @@ func main() { } defer sessionResolver.CloseIdleConnections() - geolocateConfig := &geolocate.Config{ + geolocateConfig := geolocate.Config{ Resolver: sessionResolver, Logger: logger, UserAgent: model.HTTPHeaderUserAgent, } - geolocateTask := geolocate.NewTask(*geolocateConfig) // XXX + geolocateTask := geolocate.NewTask(geolocateConfig) // XXX location := runtimex.Try1(geolocateTask.Run(ctx)) logger.Infof("%+v", location) From 4f1a2172bc52722905008fd0855184710d9e22ff Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 8 Feb 2023 07:17:18 +0000 Subject: [PATCH 08/17] refactor: move Measurement constructor inside model --- .../dismantle/backendclient/measurement.go | 57 ------------------- internal/cmd/dismantle/main.go | 2 +- internal/model/measurement.go | 49 ++++++++++++++++ 3 files changed, 50 insertions(+), 58 deletions(-) delete mode 100644 internal/cmd/dismantle/backendclient/measurement.go diff --git a/internal/cmd/dismantle/backendclient/measurement.go b/internal/cmd/dismantle/backendclient/measurement.go deleted file mode 100644 index eaacbe0db0..0000000000 --- a/internal/cmd/dismantle/backendclient/measurement.go +++ /dev/null @@ -1,57 +0,0 @@ -package backendclient - -import ( - "runtime" - "time" - - "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/platform" - "github.com/ooni/probe-cli/v3/internal/runtimex" - "github.com/ooni/probe-cli/v3/internal/version" -) - -// dateFormat is the data format used to fill a measurement. -const dateFormat = "2006-01-02 15:04:05" - -// NewMeasurement constructs a new measurement. -func NewMeasurement( - location model.LocationProvider, - testName string, - testVersion string, - testStartTime time.Time, - reportID string, - softwareName string, - softwareVersion string, - input string, -) *model.Measurement { - utctimenow := time.Now().UTC() - m := &model.Measurement{ - DataFormatVersion: model.OOAPIReportDefaultDataFormatVersion, - Input: model.MeasurementTarget(input), - MeasurementStartTime: utctimenow.Format(dateFormat), - MeasurementStartTimeSaved: utctimenow, - ProbeIP: model.DefaultProbeIP, - ProbeASN: location.ProbeASNString(), - ProbeCC: location.ProbeCC(), - ProbeNetworkName: location.ProbeNetworkName(), - ReportID: reportID, - ResolverASN: location.ResolverASNString(), - ResolverIP: location.ResolverIP(), - ResolverNetworkName: location.ResolverNetworkName(), - SoftwareName: softwareName, - SoftwareVersion: softwareVersion, - TestName: testName, - TestStartTime: testStartTime.Format(dateFormat), - TestVersion: testVersion, - } - m.AddAnnotation("architecture", runtime.GOARCH) - m.AddAnnotation("engine_name", "ooniprobe-engine") - m.AddAnnotation("engine_version", version.Version) - m.AddAnnotation("go_version", runtimex.BuildInfo.GoVersion) - m.AddAnnotation("platform", platform.Name()) - m.AddAnnotation("vcs_modified", runtimex.BuildInfo.VcsModified) - m.AddAnnotation("vcs_revision", runtimex.BuildInfo.VcsRevision) - m.AddAnnotation("vcs_time", runtimex.BuildInfo.VcsTime) - m.AddAnnotation("vcs_tool", runtimex.BuildInfo.VcsTool) - return m -} diff --git a/internal/cmd/dismantle/main.go b/internal/cmd/dismantle/main.go index 8f2dc99012..1f48d05e24 100644 --- a/internal/cmd/dismantle/main.go +++ b/internal/cmd/dismantle/main.go @@ -122,7 +122,7 @@ func main() { for _, input := range checkInResult.Tests.WebConnectivity.URLs { cfg := &webconnectivitylte.Config{} runner := webconnectivitylte.NewExperimentMeasurer(cfg) - measurement := backendclient.NewMeasurement( + measurement := model.NewMeasurement( location, runner.ExperimentName(), runner.ExperimentVersion(), testStartTime, reportID, softwareName, softwareVersion, input.URL, ) diff --git a/internal/model/measurement.go b/internal/model/measurement.go index 7cd2a8e13e..aad271da22 100644 --- a/internal/model/measurement.go +++ b/internal/model/measurement.go @@ -10,9 +10,12 @@ import ( "errors" "fmt" "net" + "runtime" "time" + "github.com/ooni/probe-cli/v3/internal/platform" "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/version" ) const ( @@ -215,3 +218,49 @@ func scrubTestKeys(m *Measurement, currentIP string) error { data = bytes.ReplaceAll(data, []byte(currentIP), []byte(Scrubbed)) return scrubJSONUnmarshalTestKeys(data, &m.TestKeys) } + +// dateFormat is the data format used to fill a measurement. +const dateFormat = "2006-01-02 15:04:05" + +// NewMeasurement constructs a new measurement. +func NewMeasurement( + location LocationProvider, + testName string, + testVersion string, + testStartTime time.Time, + reportID string, + softwareName string, + softwareVersion string, + input string, +) *Measurement { + utctimenow := time.Now().UTC() + m := &Measurement{ + DataFormatVersion: OOAPIReportDefaultDataFormatVersion, + Input: MeasurementTarget(input), + MeasurementStartTime: utctimenow.Format(dateFormat), + MeasurementStartTimeSaved: utctimenow, + ProbeIP: DefaultProbeIP, + ProbeASN: location.ProbeASNString(), + ProbeCC: location.ProbeCC(), + ProbeNetworkName: location.ProbeNetworkName(), + ReportID: reportID, + ResolverASN: location.ResolverASNString(), + ResolverIP: location.ResolverIP(), + ResolverNetworkName: location.ResolverNetworkName(), + SoftwareName: softwareName, + SoftwareVersion: softwareVersion, + TestName: testName, + TestStartTime: testStartTime.Format(dateFormat), + TestVersion: testVersion, + } + m.AddAnnotation("architecture", runtime.GOARCH) + m.AddAnnotation("engine_name", "ooniprobe-engine") + m.AddAnnotation("engine_version", version.Version) + m.AddAnnotation("go_version", runtimex.BuildInfo.GoVersion) + m.AddAnnotation("platform", platform.Name()) + m.AddAnnotation("vcs_modified", runtimex.BuildInfo.VcsModified) + m.AddAnnotation("vcs_revision", runtimex.BuildInfo.VcsRevision) + m.AddAnnotation("vcs_time", runtimex.BuildInfo.VcsTime) + m.AddAnnotation("vcs_tool", runtimex.BuildInfo.VcsTool) + return m +} From e44839b53ca7f22b1babc6d6f62a1e55b0fb38b2 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 8 Feb 2023 07:19:28 +0000 Subject: [PATCH 09/17] refactor: move descriptor creation into ooapi --- internal/cmd/dismantle/backendclient/backendclient.go | 2 +- .../backendclient/submitter.go => ooapi/submit.go} | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) rename internal/{cmd/dismantle/backendclient/submitter.go => ooapi/submit.go} (85%) diff --git a/internal/cmd/dismantle/backendclient/backendclient.go b/internal/cmd/dismantle/backendclient/backendclient.go index 096c52c37a..dead99bb04 100644 --- a/internal/cmd/dismantle/backendclient/backendclient.go +++ b/internal/cmd/dismantle/backendclient/backendclient.go @@ -61,7 +61,7 @@ func (c *Client) Submit(ctx context.Context, m *model.Measurement) error { Format: "json", Content: m, } - descriptor := newSubmitDescriptor(req, m.ReportID) + descriptor := ooapi.NewSubmitMeasurementDescriptor(req, m.ReportID) _, err := httpapi.Call(ctx, descriptor, c.endpoint) return err } diff --git a/internal/cmd/dismantle/backendclient/submitter.go b/internal/ooapi/submit.go similarity index 85% rename from internal/cmd/dismantle/backendclient/submitter.go rename to internal/ooapi/submit.go index 8bfed04b2d..f6327dd69f 100644 --- a/internal/cmd/dismantle/backendclient/submitter.go +++ b/internal/ooapi/submit.go @@ -1,4 +1,4 @@ -package backendclient +package ooapi import ( "encoding/json" @@ -10,7 +10,9 @@ import ( "github.com/ooni/probe-cli/v3/internal/runtimex" ) -func newSubmitDescriptor( +// NewSubmitMeasurementDescriptor creates a new [httpapi.Descriptor] describing how +// to submit a measurement to the OONI backend. +func NewSubmitMeasurementDescriptor( req *model.OOAPICollectorUpdateRequest, reportID string) *httpapi.Descriptor[ *model.OOAPICollectorUpdateRequest, *model.OOAPICollectorUpdateResponse] { rawBody, err := json.Marshal(req) From 888a94ec616de1bfb968589b60bcb0e38f564400 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 8 Feb 2023 07:23:59 +0000 Subject: [PATCH 10/17] refactor: document and improve backendclient implementation --- .../dismantle/backendclient/backendclient.go | 35 ++++++++++++++----- internal/cmd/dismantle/main.go | 1 - 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/internal/cmd/dismantle/backendclient/backendclient.go b/internal/cmd/dismantle/backendclient/backendclient.go index dead99bb04..a6bbe87227 100644 --- a/internal/cmd/dismantle/backendclient/backendclient.go +++ b/internal/cmd/dismantle/backendclient/backendclient.go @@ -1,7 +1,10 @@ +// Package backendclient implements a client to communicate +// with the OONI backend infrastructure. package backendclient import ( "context" + "errors" "net/url" "github.com/ooni/probe-cli/v3/internal/httpapi" @@ -9,21 +12,31 @@ import ( "github.com/ooni/probe-cli/v3/internal/ooapi" ) +// Config contains configuration for [New]. type Config struct { - KVStore model.KeyValueStore + // BaseURL is the OPTIONAL OONI backend URL. + BaseURL *url.URL + + // KVStore is the MANDATORY key-value store to use. + KVStore model.KeyValueStore + + // HTTPClient is the MANDATORY underlying HTTPClient to use. HTTPClient model.HTTPClient - Logger model.Logger - UserAgent string - // optional fields - BaseURL *url.URL - ProxyURL *url.URL + // Logger is the MANDATORY logger to use. + Logger model.Logger + + // UserAgent is the MANDATORY user agent to use. + UserAgent string } +// Client is a client to communicate with the OONI backend. type Client struct { + // endpoint is the HTTP API endpoint. endpoint *httpapi.Endpoint } +// New constructs a new instance of [Client]. func New(config *Config) *Client { baseURL := "https://api.ooni.io/" if config.BaseURL != nil { @@ -32,7 +45,7 @@ func New(config *Config) *Client { endpoint := &httpapi.Endpoint{ BaseURL: baseURL, HTTPClient: config.HTTPClient, - Host: "", + Host: "", // no need to configure Logger: config.Logger, UserAgent: config.UserAgent, } @@ -42,20 +55,24 @@ func New(config *Config) *Client { return backendClient } +// CheckIn invokes the check-in API. func (c *Client) CheckIn( ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResult, error) { return httpapi.Call(ctx, ooapi.NewDescriptorCheckIn(config), c.endpoint) } +// FetchPsiphonConfig retrieves Psiphon configuration. func (c *Client) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { - panic("not implemented") + return nil, errors.New("not implemented") } +// FetchTorTargets fetches measurement targets for the tor experiment. func (c *Client) FetchTorTargets( ctx context.Context, cc string) (result map[string]model.OOAPITorTarget, err error) { - panic("not implemented") + return nil, errors.New("not implemented") } +// Submit submits the given measurement. func (c *Client) Submit(ctx context.Context, m *model.Measurement) error { req := &model.OOAPICollectorUpdateRequest{ Format: "json", diff --git a/internal/cmd/dismantle/main.go b/internal/cmd/dismantle/main.go index 1f48d05e24..d311906c4f 100644 --- a/internal/cmd/dismantle/main.go +++ b/internal/cmd/dismantle/main.go @@ -87,7 +87,6 @@ func main() { Logger: logger, UserAgent: userAgent, BaseURL: nil, - ProxyURL: proxyURL, } backendClient := backendclient.New(backendClientConfig) From 5204b1369ca8868cc43cf6a615a9a57902801b96 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 8 Feb 2023 07:24:39 +0000 Subject: [PATCH 11/17] refactor: make backendclient a toplevel package --- internal/{cmd/dismantle => }/backendclient/backendclient.go | 0 internal/cmd/dismantle/main.go | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename internal/{cmd/dismantle => }/backendclient/backendclient.go (100%) diff --git a/internal/cmd/dismantle/backendclient/backendclient.go b/internal/backendclient/backendclient.go similarity index 100% rename from internal/cmd/dismantle/backendclient/backendclient.go rename to internal/backendclient/backendclient.go diff --git a/internal/cmd/dismantle/main.go b/internal/cmd/dismantle/main.go index d311906c4f..43f12b26a5 100644 --- a/internal/cmd/dismantle/main.go +++ b/internal/cmd/dismantle/main.go @@ -9,8 +9,8 @@ import ( "time" "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/backendclient" "github.com/ooni/probe-cli/v3/internal/bytecounter" - "github.com/ooni/probe-cli/v3/internal/cmd/dismantle/backendclient" "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivitylte" "github.com/ooni/probe-cli/v3/internal/geolocate" "github.com/ooni/probe-cli/v3/internal/kvstore" From 1585ec9ceefc11925d3a4e075c785b5cf3f73c0e Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 8 Feb 2023 14:56:05 +0000 Subject: [PATCH 12/17] it works(TM) --- internal/cmd/dismantle2/main.go | 79 +++++++++ internal/engine/session.go | 2 +- internal/engine/session_nopsiphon.go | 6 +- internal/engine/session_nopsiphon_test.go | 2 +- internal/engine/session_psiphon.go | 10 +- internal/netxlite/maybeproxy.go | 2 + internal/session/bootstrap.go | 91 ++++++++++ internal/session/checkin.go | 51 ++++++ internal/session/client.go | 172 ++++++++++++++++++ internal/session/geolocate.go | 67 +++++++ internal/session/logger.go | 81 +++++++++ internal/session/progressbar.go | 43 +++++ internal/session/session.go | 202 ++++++++++++++++++++++ internal/session/sessionadapter.go | 111 ++++++++++++ internal/session/state.go | 190 ++++++++++++++++++++ internal/session/submit.go | 40 +++++ internal/session/tickerservice.go | 59 +++++++ internal/session/tunnel.go | 118 +++++++++++++ internal/session/webconnectivity.go | 85 +++++++++ internal/sessionresolver/resolver.go | 2 + 20 files changed, 1403 insertions(+), 10 deletions(-) create mode 100644 internal/cmd/dismantle2/main.go create mode 100644 internal/session/bootstrap.go create mode 100644 internal/session/checkin.go create mode 100644 internal/session/client.go create mode 100644 internal/session/geolocate.go create mode 100644 internal/session/logger.go create mode 100644 internal/session/progressbar.go create mode 100644 internal/session/session.go create mode 100644 internal/session/sessionadapter.go create mode 100644 internal/session/state.go create mode 100644 internal/session/submit.go create mode 100644 internal/session/tickerservice.go create mode 100644 internal/session/tunnel.go create mode 100644 internal/session/webconnectivity.go diff --git a/internal/cmd/dismantle2/main.go b/internal/cmd/dismantle2/main.go new file mode 100644 index 0000000000..3d91bef5fc --- /dev/null +++ b/internal/cmd/dismantle2/main.go @@ -0,0 +1,79 @@ +package main + +import ( + "context" + "path/filepath" + "time" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/platform" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/session" +) + +const softwareName = "dismantle" + +const softwareVersion = "0.1.0-dev" + +func main() { + log.SetLevel(log.DebugLevel) + + client := session.NewClient(log.Log) + ctx := context.Background() + + bootstrapRequest := &session.BootstrapRequest{ + SnowflakeRendezvousMethod: "", + StateDir: filepath.Join("testdata", "engine"), + ProxyURL: "psiphon:///", + SoftwareName: softwareName, + SoftwareVersion: softwareVersion, + TorArgs: nil, + TorBinary: "", + TempDir: filepath.Join("testdata", "tmp"), + TunnelDir: filepath.Join("testdata", "tunnel"), + VerboseLogging: false, + } + runtimex.Try0(client.Bootstrap(ctx, bootstrapRequest)) + + geolocateRequest := &session.GeolocateRequest{} + location := runtimex.Try1(client.Geolocate(ctx, geolocateRequest)) + log.Infof("%+v", location) + + checkInRequest := &session.CheckInRequest{ + Charging: true, + OnWiFi: true, + Platform: platform.Name(), + ProbeASN: location.ProbeASNString(), + ProbeCC: location.ProbeCC(), + RunType: "manual", + SoftwareName: softwareName, + SoftwareVersion: softwareVersion, + WebConnectivity: model.OOAPICheckInConfigWebConnectivity{ + CategoryCodes: []string{}, + }, + } + checkInResponse := runtimex.Try1(client.CheckIn(ctx, checkInRequest)) + log.Infof("%+v", checkInResponse) + + runtimex.Assert(checkInResponse.Tests.WebConnectivity != nil, "no web connectivity info") + + reportID := checkInResponse.Tests.WebConnectivity.ReportID + testStartTime := time.Now() + for _, entry := range checkInResponse.Tests.WebConnectivity.URLs { + webConnectivityRequest := &session.WebConnectivityRequest{ + Input: entry.URL, + ReportID: reportID, + TestStartTime: testStartTime, + } + measurement, err := client.WebConnectivity(ctx, webConnectivityRequest) + if err != nil { + log.Warnf("webconnectivity: measure: %s", err.Error()) + continue + } + if err := client.Submit(ctx, measurement); err != nil { + log.Warnf("webconnectivity: submit: %s", err.Error()) + continue + } + } +} diff --git a/internal/engine/session.go b/internal/engine/session.go index 010abededa..a8249b69d2 100644 --- a/internal/engine/session.go +++ b/internal/engine/session.go @@ -191,7 +191,7 @@ func NewSession(ctx context.Context, config SessionConfig) (*Session, error) { Logger: config.Logger, Name: proxyURL.Scheme, SnowflakeRendezvous: config.SnowflakeRendezvous, - Session: &sessionTunnelEarlySession{}, + Session: &SessionTunnelEarlySession{}, TorArgs: config.TorArgs, TorBinary: config.TorBinary, TunnelDir: config.TunnelDir, diff --git a/internal/engine/session_nopsiphon.go b/internal/engine/session_nopsiphon.go index cc03326d57..16683ae5ea 100644 --- a/internal/engine/session_nopsiphon.go +++ b/internal/engine/session_nopsiphon.go @@ -16,16 +16,16 @@ func (s *Session) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { return clnt.FetchPsiphonConfig(ctx) } -// sessionTunnelEarlySession is the early session that we pass +// SessionTunnelEarlySession is the early session that we pass // to tunnel.Start to fetch the Psiphon configuration. -type sessionTunnelEarlySession struct{} +type SessionTunnelEarlySession struct{} // errPsiphonNoEmbeddedConfig indicates that there is no // embedded psiphong config file in this binary. var errPsiphonNoEmbeddedConfig = errors.New("no embedded configuration file") // FetchPsiphonConfig implements tunnel.Session.FetchPsiphonConfig. -func (s *sessionTunnelEarlySession) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { +func (s *SessionTunnelEarlySession) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { return nil, errPsiphonNoEmbeddedConfig } diff --git a/internal/engine/session_nopsiphon_test.go b/internal/engine/session_nopsiphon_test.go index ecb6cc75a4..99e013e7da 100644 --- a/internal/engine/session_nopsiphon_test.go +++ b/internal/engine/session_nopsiphon_test.go @@ -9,7 +9,7 @@ import ( ) func TestEarlySessionNoPsiphonFetchPsiphonConfig(t *testing.T) { - s := &sessionTunnelEarlySession{} + s := &SessionTunnelEarlySession{} out, err := s.FetchPsiphonConfig(context.Background()) if !errors.Is(err, errPsiphonNoEmbeddedConfig) { t.Fatal("not the error we expected", err) diff --git a/internal/engine/session_psiphon.go b/internal/engine/session_psiphon.go index 48b5ab0a06..bd4651761a 100644 --- a/internal/engine/session_psiphon.go +++ b/internal/engine/session_psiphon.go @@ -17,13 +17,13 @@ var psiphonConfigJSONAge []byte //go:embed psiphon-config.key var psiphonConfigSecretKey string -// sessionTunnelEarlySession is the early session that we pass +// SessionTunnelEarlySession is the early session that we pass // to tunnel.Start to fetch the Psiphon configuration. -type sessionTunnelEarlySession struct{} +type SessionTunnelEarlySession struct{} // FetchPsiphonConfig decrypts psiphonConfigJSONAge using // filippo.io/age _and_ psiphonConfigSecretKey. -func (s *sessionTunnelEarlySession) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { +func (s *SessionTunnelEarlySession) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { key := "AGE-SECRET-KEY-1" + psiphonConfigSecretKey identity, err := age.ParseX25519Identity(key) if err != nil { @@ -40,13 +40,13 @@ func (s *sessionTunnelEarlySession) FetchPsiphonConfig(ctx context.Context) ([]b // FetchPsiphonConfig decrypts psiphonConfigJSONAge using // filippo.io/age _and_ psiphonConfigSecretKey. func (s *Session) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { - child := &sessionTunnelEarlySession{} + child := &SessionTunnelEarlySession{} return child.FetchPsiphonConfig(ctx) } // CheckEmbeddedPsiphonConfig checks whether we can load psiphon's config func CheckEmbeddedPsiphonConfig() error { - child := &sessionTunnelEarlySession{} + child := &SessionTunnelEarlySession{} _, err := child.FetchPsiphonConfig(context.Background()) return err } diff --git a/internal/netxlite/maybeproxy.go b/internal/netxlite/maybeproxy.go index edab70b19b..ceea868f76 100644 --- a/internal/netxlite/maybeproxy.go +++ b/internal/netxlite/maybeproxy.go @@ -7,6 +7,7 @@ package netxlite import ( "context" "errors" + "log" "net" "net/url" @@ -44,6 +45,7 @@ var ErrProxyUnsupportedScheme = errors.New("proxy: unsupported scheme") // DialContext implements Dialer.DialContext. func (d *proxyDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + log.Printf("HEY dialing with %s", d.ProxyURL) url := d.ProxyURL if url.Scheme != "socks5" { return nil, ErrProxyUnsupportedScheme diff --git a/internal/session/bootstrap.go b/internal/session/bootstrap.go new file mode 100644 index 0000000000..a0567b459b --- /dev/null +++ b/internal/session/bootstrap.go @@ -0,0 +1,91 @@ +package session + +import ( + "context" + "errors" +) + +// BootstrapRequest is a request to bootstrap the [Session] and +// contains the arguments requested by the bootstrap. You can +// boostrap a [Session] just once. All operations you would like +// to perform with a [Session] require a boostrap first. +type BootstrapRequest struct { + // SnowflakeRendezvousMethod is the OPTIONAL rendezvous + // method to use when ProxyURL scheme is `torsf`. + SnowflakeRendezvousMethod string + + // StateDir is the MANDATORY directory where to store + // persistent engine state using a key-value store. + StateDir string + + // ProxyURL is the OPTIONAL proxy URL. We accept the + // following proxy URL schemes: + // + // - "socks5": configures a socks5 proxy; + // + // - "tor": requires a tor tunnel; + // + // - "torsf": requires a tor+snowflake tunnel; + // + // - "psiphon": requires a psiphon tunnel. + // + // When requesting a tunnel, we only check the URL scheme + // and disregard the rest of the URL. + ProxyURL string + + // SoftwareName is the MANDATORY software name. + SoftwareName string + + // SoftwareVersion is the MANDATORY software version. + SoftwareVersion string + + // TorArgs OPTIONALLY passes command line arguments to tor + // when the ProxyURL scheme is "tor" or "torsf". + TorArgs []string + + // TorBinary OPTIONALLY tells the engine to use a specific + // binary for starting the "tor" and "torsf" tunnels. + TorBinary string + + // TempDir is the MANDATORY base directory in which + // the session should store temporary state. + TempDir string + + // TunnelDir is the MANDATORY directory in which + // to store persistent tunnel state. + TunnelDir string + + // VerboseLogging OPTIONALLY enables verbose logging. + VerboseLogging bool +} + +// BootstrapEvent is the event emmitted at the end of the bootstrap. +type BootstrapEvent struct { + // Error is the bootstrap result. + Error error +} + +// boostrap bootstraps a session. +func (s *Session) bootstrap(ctx context.Context, req *Request) { + s.emit(&Event{ + Bootstrap: &BootstrapEvent{ + Error: s.dobootstrap(ctx, req), + }, + }) +} + +// ErrAlreadyBootstrapped indicates that we already bootstrapped a [Session]. +var ErrAlreadyBootstrapped = errors.New("session: already bootstrapped") + +// dobootstrap implements bootstrap. +func (s *Session) dobootstrap(ctx context.Context, req *Request) error { + if s.state != nil { + return ErrAlreadyBootstrapped + } + state, err := s.newState(ctx, req) + if err != nil { + return err + } + s.state = state + return nil +} diff --git a/internal/session/checkin.go b/internal/session/checkin.go new file mode 100644 index 0000000000..019e687761 --- /dev/null +++ b/internal/session/checkin.go @@ -0,0 +1,51 @@ +package session + +import ( + "context" + + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// CheckInRequest is the request for the check-in API. +type CheckInRequest = model.OOAPICheckInConfig + +// CheckInEvent is the result of calling the check-in API. +type CheckInEvent struct { + // Error is the geolocate result. + Error error + + // Result is the result returned on success. + Result *model.OOAPICheckInResult +} + +// checkin calls the check-in API. +func (s *Session) checkin(ctx context.Context, req *Request) { + result, err := s.docheckin(ctx, req) + event := &Event{ + CheckIn: &CheckInEvent{ + Error: err, + Result: result, + }, + } + s.emit(event) +} + +// docheckin implements checkin. +func (s *Session) docheckin(ctx context.Context, req *Request) (*model.OOAPICheckInResult, error) { + runtimex.Assert(req.CheckIn != nil, "passed a nil CheckIn") + + if s.state == nil { + return nil, ErrNotBootstrapped + } + + ts := newTickerService(ctx, s) + defer ts.stop() + + result, err := s.state.backendClient.CheckIn(ctx, req.CheckIn) + if err != nil { + return nil, err + } + s.state.checkIn = result + return result, nil +} diff --git a/internal/session/client.go b/internal/session/client.go new file mode 100644 index 0000000000..3c3ba10a1d --- /dev/null +++ b/internal/session/client.go @@ -0,0 +1,172 @@ +package session + +import ( + "context" + "sync" + + "github.com/ooni/probe-cli/v3/internal/geolocate" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// Client is a client for a [Session]. You should use a [Client] +// when you are writing Go code that uses a [Session]. +type Client struct { + logger model.Logger + once sync.Once + session *Session +} + +// NewClient creates a [Client] instance. +func NewClient(logger model.Logger) *Client { + return &Client{ + logger: logger, + once: sync.Once{}, + session: New(), + } +} + +// Bootstrap bootstraps a session. +func (c *Client) Bootstrap(ctx context.Context, req *BootstrapRequest) error { + if err := c.session.Send(ctx, &Request{Bootstrap: req}); err != nil { + return err + } + for { + resp, err := c.session.Recv(ctx) + if err != nil { + return err + } + if resp.Log != nil { + c.emitLog(resp.Log) + continue + } + if resp.Ticker != nil { + c.logger.Infof("bootstrap in progress (elapsed: %+v)", resp.Ticker.ElapsedTime) + continue + } + if resp.Bootstrap != nil { + return resp.Bootstrap.Error + } + c.logger.Warnf("unexpected event: %+v", resp) + } +} + +// Geolocate performs geolocation. You must run bootstrap first. +func (c *Client) Geolocate(ctx context.Context, req *GeolocateRequest) (*geolocate.Results, error) { + if err := c.session.Send(ctx, &Request{Geolocate: req}); err != nil { + return nil, err + } + for { + resp, err := c.session.Recv(ctx) + if err != nil { + return nil, err + } + if resp.Log != nil { + c.emitLog(resp.Log) + continue + } + if resp.Ticker != nil { + c.logger.Infof("geolocate in progress (elapsed: %+v)", resp.Ticker.ElapsedTime) + continue + } + if resp.Geolocate != nil { + return resp.Geolocate.Location, resp.Geolocate.Error + } + c.logger.Warnf("unexpected event: %+v", resp) + } +} + +// CheckIn calls the check-in API. You must run bootstrap first. +func (c *Client) CheckIn(ctx context.Context, req *CheckInRequest) (*model.OOAPICheckInResult, error) { + if err := c.session.Send(ctx, &Request{CheckIn: req}); err != nil { + return nil, err + } + for { + resp, err := c.session.Recv(ctx) + if err != nil { + return nil, err + } + if resp.Log != nil { + c.emitLog(resp.Log) + continue + } + if resp.Ticker != nil { + c.logger.Infof("check-in in progress (elapsed: %+v)", resp.Ticker.ElapsedTime) + continue + } + if resp.CheckIn != nil { + return resp.CheckIn.Result, resp.CheckIn.Error + } + c.logger.Warnf("unexpected event: %+v", resp) + } +} + +// WebConnectivity runs a single-URL Web Connectivity measurement. +func (c *Client) WebConnectivity( + ctx context.Context, req *WebConnectivityRequest) (*model.Measurement, error) { + if err := c.session.Send(ctx, &Request{WebConnectivity: req}); err != nil { + return nil, err + } + for { + resp, err := c.session.Recv(ctx) + if err != nil { + return nil, err + } + if resp.Log != nil { + c.emitLog(resp.Log) + continue + } + if resp.Ticker != nil { + c.logger.Infof("webconnectivity (elapsed: %+v)", resp.Ticker.ElapsedTime) + continue + } + if resp.WebConnectivity != nil { + return resp.WebConnectivity.Measurement, resp.WebConnectivity.Error + } + c.logger.Warnf("unexpected event: %+v", resp) + } +} + +// Submit submits a measurement. +func (c *Client) Submit(ctx context.Context, measurement *model.Measurement) error { + if err := c.session.Send(ctx, &Request{Submit: measurement}); err != nil { + return err + } + for { + resp, err := c.session.Recv(ctx) + if err != nil { + return err + } + if resp.Log != nil { + c.emitLog(resp.Log) + continue + } + if resp.Ticker != nil { + c.logger.Infof("webconnectivity (elapsed: %+v)", resp.Ticker.ElapsedTime) + continue + } + if resp.Submit != nil { + return resp.Submit.Error + } + c.logger.Warnf("unexpected event: %+v", resp) + } +} + +// Close releases the resources used by a [Client]. +func (c *Client) Close() (err error) { + c.once.Do(func() { + err = c.session.Close() + }) + return +} + +// emitLog emits a log event. +func (c *Client) emitLog(ev *LogEvent) { + switch ev.Level { + case "DEBUG": + c.logger.Debug(ev.Message) + case "WARNING": + c.logger.Warn(ev.Message) + default: + c.logger.Info(ev.Message) + } +} diff --git a/internal/session/geolocate.go b/internal/session/geolocate.go new file mode 100644 index 0000000000..cbc606cb1f --- /dev/null +++ b/internal/session/geolocate.go @@ -0,0 +1,67 @@ +package session + +import ( + "context" + "errors" + + "github.com/ooni/probe-cli/v3/internal/geolocate" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// GeolocateRequest contains config for geolocate. +type GeolocateRequest struct{} + +// GeolocateEvent is the event emitted at the end of geolocate. +type GeolocateEvent struct { + // Error is the geolocate result. + Error error + + // Location is the probe location. + Location *geolocate.Results +} + +// geolocate performs a geolocation. +func (s *Session) geolocate(ctx context.Context, req *Request) { + location, err := s.dogeolocate(ctx, req) + event := &Event{ + Geolocate: &GeolocateEvent{ + Error: err, + Location: location, + }, + } + s.emit(event) +} + +// ErrNotBootstrapped indicates we didn't bootstrap a session. +var ErrNotBootstrapped = errors.New("session: not bootstrapped") + +// dogeolocate implements geolocate. +func (s *Session) dogeolocate(ctx context.Context, req *Request) (*geolocate.Results, error) { + runtimex.Assert(req.Geolocate != nil, "passed a nil Geolocate") + + if s.state == nil { + return nil, ErrNotBootstrapped + } + + ts := newTickerService(ctx, s) + defer ts.stop() + + geolocateConfig := geolocate.Config{ + Resolver: s.state.resolver, + Logger: s.state.logger, + UserAgent: model.HTTPHeaderUserAgent, + } + task := geolocate.NewTask(geolocateConfig) // XXX + + // TODO(bassosimone): we should make geolocate.Results a type + // in the internal/model package and use the ~typical naming for + // its fields rather than the naming we have here. + location, err := task.Run(ctx) + if err != nil { + return nil, err + } + + s.state.location = location + return location, nil +} diff --git a/internal/session/logger.go b/internal/session/logger.go new file mode 100644 index 0000000000..8cb970c326 --- /dev/null +++ b/internal/session/logger.go @@ -0,0 +1,81 @@ +package session + +import ( + "fmt" + "time" + + "github.com/ooni/probe-cli/v3/internal/model" +) + +// LogEvent is an event emitted by a logger. +type LogEvent struct { + // Timestamp is the log timestamp. + Timestamp time.Time + + // Level is the log level. + Level string + + // Message is the log message. + Message string +} + +// newLogger creates a [model.Logger] that emits +// [LogEvent] events using the [Session]. +func (s *Session) newLogger(verbose bool) model.Logger { + return &sessionLogger{ + session: s, + verbose: verbose, + } +} + +// sessionLogger is a [model.Logger] using a [Session]. +type sessionLogger struct { + session *Session + verbose bool +} + +// Debug implements model.Logger +func (sl *sessionLogger) Debug(msg string) { + if sl.verbose { + sl.emit("DEBUG", msg) + } +} + +// Debugf implements model.Logger +func (sl *sessionLogger) Debugf(format string, v ...interface{}) { + if sl.verbose { + sl.Debug(fmt.Sprintf(format, v...)) + } +} + +// Info implements model.Logger +func (sl *sessionLogger) Info(msg string) { + sl.emit("INFO", msg) +} + +// Infof implements model.Logger +func (sl *sessionLogger) Infof(format string, v ...interface{}) { + sl.Info(fmt.Sprintf(format, v...)) +} + +// Warn implements model.Logger +func (sl *sessionLogger) Warn(msg string) { + sl.emit("WARNING", msg) +} + +// Warnf implements model.Logger +func (sl *sessionLogger) Warnf(format string, v ...interface{}) { + sl.Warn(fmt.Sprintf(format, v...)) +} + +// emit emits a log message. +func (sl *sessionLogger) emit(level, message string) { + ev := &Event{ + Log: &LogEvent{ + Timestamp: time.Now(), + Level: level, + Message: message, + }, + } + sl.session.emit(ev) +} diff --git a/internal/session/progressbar.go b/internal/session/progressbar.go new file mode 100644 index 0000000000..828e41761b --- /dev/null +++ b/internal/session/progressbar.go @@ -0,0 +1,43 @@ +package session + +import ( + "time" + + "github.com/ooni/probe-cli/v3/internal/model" +) + +// ProgressEvent indicates the progress in a task completion. +type ProgressEvent struct { + // Timestamp is the timestamp. + Timestamp time.Time + + // Completion is a number between 0 and 1. + Completion float64 + + // Message is the message. + Message string +} + +// progressBar emits progress. +type progressBar struct { + session *Session +} + +var _ model.ExperimentCallbacks = &progressBar{} + +// OnProgress implements model.ExperimentCallbacks +func (pb *progressBar) OnProgress(completion float64, message string) { + pb.emit(completion, message) +} + +// emit emits a progress event. +func (pb *progressBar) emit(completion float64, message string) { + ev := &Event{ + Progress: &ProgressEvent{ + Timestamp: time.Now(), + Completion: completion, + Message: message, + }, + } + pb.session.emit(ev) +} diff --git a/internal/session/session.go b/internal/session/session.go new file mode 100644 index 0000000000..502589abb9 --- /dev/null +++ b/internal/session/session.go @@ -0,0 +1,202 @@ +// Package session implements a measurement session. The design of +// this package is such that we can split the measurement engine proper +// and the application using it. In particular, this design is such +// that it would be easy to expose this API as a C library. +// +// The general usage of this package is the following: +// +// 1. XXX document +// +// Go packages should not use this package directly but rather use +// the sessionclient package, which provides idiomatic wrappers. +package session + +import ( + "context" + "errors" + "log" + "sync" +) + +// Session is a measurement session. +type Session struct { + // cancel allows us to terminate the backround goroutine. + cancel context.CancelFunc + + // input is the channel to send input to the background goroutine. + input chan *Request + + // once allows us to run cleanups just once. + once sync.Once + + // output is the channel from which we read the emitted events. + output chan *Event + + // state is the background goroutine's state, which only + // the background goroutine is allowed to modify. + state *state + + // terminated is closed when the background goroutine terminates. + terminated chan any +} + +// New creates a new measurement [Session]. This function will create +// a background goroutine that will handle incoming [Request]s. +func New() *Session { + ctx, cancel := context.WithCancel(context.Background()) + s := &Session{ + cancel: cancel, + input: make(chan *Request, 1024), + once: sync.Once{}, + output: make(chan *Event, 1024), + state: nil, + terminated: make(chan any), + } + go s.mainloop(ctx) + return s +} + +// Request requests a [Session] to perform a background task. This +// struct contains several pointers. Each of them indicates a specific +// task the [Session] should run. The [Session] will go through each +// pointer and only consider the first one that is not nil. +type Request struct { + // Bootstrap indicates that the [Session] should bootstrap + // and contains bootstrap configuration. + Bootstrap *BootstrapRequest + + // CheckIn indicates that the [Session] should call the check-in API. + CheckIn *CheckInRequest + + // Geolocate indicates that the [Session] should obtain + // the current probe's geolocation. + Geolocate *GeolocateRequest + + // Submit indicates that the [Session] should submit a measurement. + Submit *SubmitRequest + + // WebConnectivity indicates that the [Session] should + // run the Web Connectivity experiment. + WebConnectivity *WebConnectivityRequest +} + +// ErrSessionTerminated indicates that the background goroutine +// servicing the [Session] [Request]s has stopped. +var ErrSessionTerminated = errors.New("session: terminated") + +// Send sends a [Request] to a [Session]. This function will return an +// error when the [Session] has been stopped or when the context has expired. +func (s *Session) Send(ctx context.Context, req *Request) error { + select { + case s.input <- req: + return nil + case <-s.terminated: + return ErrSessionTerminated + case <-ctx.Done(): + return ctx.Err() + } +} + +// Close stops the goroutine running in the background and release +// all the resources allocated by the [Session]. +func (s *Session) Close() error { + s.once.Do(s.joincleanup) + return nil +} + +// joincleanup joins the [Session] and closes open resources. +func (s *Session) joincleanup() { + s.cancel() + <-s.terminated + if s.state != nil { + s.state.cleanup() + } +} + +// Recv receives the next [Event] emitted by a [Session]. This function +// will return an error when the [Session] has been stopped or when +// the context has expired. +func (s *Session) Recv(ctx context.Context) (*Event, error) { + select { + case ev := <-s.output: + return ev, nil + case <-s.terminated: + return nil, ErrSessionTerminated + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +// emit emits an [Event]. +func (s *Session) emit(ev *Event) { + select { + case s.output <- ev: + default: + log.Printf("session: cannot send event: %+v", ev) + } +} + +// Event is an event emitted by a [Session]. +type Event struct { + // Bootstrap is emitted at the end of the bootstrap. + Bootstrap *BootstrapEvent + + // CheckIn is the event emitted at the end of the check-in. + CheckIn *CheckInEvent + + // Geolocate is emitted at the end of geolocate. + Geolocate *GeolocateEvent + + // Log is a log event. + Log *LogEvent + + // Progress is a progress event. + Progress *ProgressEvent + + // Submit is emitted after a measurement submission. + Submit *SubmitEvent + + // Ticker is a ticker event. + Ticker *TickerEvent + + // WebConnectivity is the Web Connectivity event. + WebConnectivity *WebConnectivityEvent +} + +// mainloop runs the [Session] main loop. +func (s *Session) mainloop(ctx context.Context) { + defer close(s.terminated) + for { + select { + case <-ctx.Done(): + return + case req := <-s.input: + s.handle(ctx, req) + } + } +} + +// handle handles an incoming [Request]. +func (s *Session) handle(ctx context.Context, req *Request) { + // TODO(bassosimone): rewrite trying to avoid all these ifs. + if req.Bootstrap != nil { + s.bootstrap(ctx, req) + return + } + if req.CheckIn != nil { + s.checkin(ctx, req) + return + } + if req.Geolocate != nil { + s.geolocate(ctx, req) + return + } + if req.Submit != nil { + s.submit(ctx, req) + return + } + if req.WebConnectivity != nil { + s.webconnectivity(ctx, req) + return + } +} diff --git a/internal/session/sessionadapter.go b/internal/session/sessionadapter.go new file mode 100644 index 0000000000..afa01de8ee --- /dev/null +++ b/internal/session/sessionadapter.go @@ -0,0 +1,111 @@ +package session + +import ( + "context" + "errors" + + "github.com/ooni/probe-cli/v3/internal/geolocate" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// sessionAdapter adapts [Session] to be a [model.ExperimentSesion]. +type sessionAdapter struct { + httpClient model.HTTPClient + location *geolocate.Results + logger model.Logger + tempDir string + testHelpers map[string][]model.OOAPIService + torBinary string + tunnelDir string + userAgent string +} + +// ErrNoLocation means we cannot proceed without knowing the probe location. +var ErrNoLocation = errors.New("session: no location information") + +// ErrNoCheckIn means we cannot proceed without the check-in API results. +var ErrNoCheckIn = errors.New("session: no check-in information") + +// newSessionAdapter creates a new [sessionAdapter] instance. +func newSessionAdapter(state *state) (*sessionAdapter, error) { + if state.location == nil { + return nil, ErrNoLocation + } + if state.checkIn == nil { + return nil, ErrNoCheckIn + } + sa := &sessionAdapter{ + httpClient: state.httpClient, + location: state.location, + logger: state.logger, + tempDir: state.tempDir, + testHelpers: state.checkIn.Conf.TestHelpers, + torBinary: state.torBinary, + tunnelDir: state.tunnelDir, + userAgent: state.userAgent, + } + return sa, nil +} + +var _ model.ExperimentSession = &sessionAdapter{} + +// DefaultHTTPClient implements model.ExperimentSession +func (es *sessionAdapter) DefaultHTTPClient() model.HTTPClient { + return es.httpClient +} + +// FetchPsiphonConfig implements model.ExperimentSession +func (es *sessionAdapter) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { + return nil, errors.New("not implemented") +} + +// FetchTorTargets implements model.ExperimentSession +func (es *sessionAdapter) FetchTorTargets(ctx context.Context, cc string) (map[string]model.OOAPITorTarget, error) { + return nil, errors.New("not implemented") +} + +// GetTestHelpersByName implements model.ExperimentSession +func (es *sessionAdapter) GetTestHelpersByName(name string) ([]model.OOAPIService, bool) { + value, found := es.testHelpers[name] + return value, found +} + +// Logger implements model.ExperimentSession +func (es *sessionAdapter) Logger() model.Logger { + return es.logger +} + +// ProbeCC implements model.ExperimentSession +func (es *sessionAdapter) ProbeCC() string { + return es.location.CountryCode +} + +// ResolverIP implements model.ExperimentSession +func (es *sessionAdapter) ResolverIP() string { + return es.location.ResolverIPAddr +} + +// TempDir implements model.ExperimentSession +func (es *sessionAdapter) TempDir() string { + return es.tempDir +} + +// TorArgs implements model.ExperimentSession +func (es *sessionAdapter) TorArgs() []string { + return []string{} // TODO(bassosimone): this field is only meaningful for bootstrap +} + +// TorBinary implements model.ExperimentSession +func (es *sessionAdapter) TorBinary() string { + return es.torBinary +} + +// TunnelDir implements model.ExperimentSession +func (es *sessionAdapter) TunnelDir() string { + return es.tunnelDir +} + +// UserAgent implements model.ExperimentSession +func (es *sessionAdapter) UserAgent() string { + return es.userAgent +} diff --git a/internal/session/state.go b/internal/session/state.go new file mode 100644 index 0000000000..df63ad7bf3 --- /dev/null +++ b/internal/session/state.go @@ -0,0 +1,190 @@ +package session + +import ( + "context" + "errors" + "fmt" + "os" + + "github.com/ooni/probe-cli/v3/internal/backendclient" + "github.com/ooni/probe-cli/v3/internal/bytecounter" + "github.com/ooni/probe-cli/v3/internal/geolocate" + "github.com/ooni/probe-cli/v3/internal/kvstore" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/sessionhttpclient" + "github.com/ooni/probe-cli/v3/internal/sessionresolver" + "github.com/ooni/probe-cli/v3/internal/tunnel" + "github.com/ooni/probe-cli/v3/internal/version" +) + +// state is the session's state. Only the background +// goroutine is allowed to manipulate it. +type state struct { + // backendClient is the backend client we're using. + backendClient *backendclient.Client + + // checkIn is the most recently fetched check-in result or nil. + checkIn *model.OOAPICheckInResult + + // counter is the bytecounter we're using. + counter *bytecounter.Counter + + // httpClient is the HTTP client we're using. + httpClient model.HTTPClient + + // kvstore is the session's key-value store. + kvstore model.KeyValueStore + + // location is the most recently resolved location or nil. + location *geolocate.Results + + // logger is the model.Logger we're using. + logger model.Logger + + // resolver is the session resolver. + resolver model.Resolver + + // softwareName is the software name to use. + softwareName string + + // softwareVersion is the software version to use. + softwareVersion string + + // tempDir is the session specific temporary directory. + tempDir string + + // torBinary contains the location of the tor binary to use. + torBinary string + + // torArgs should not be exposed here because we only + // want to use it for bootstrapping tor. + //torArgs []string + + // tunnelDir is the directory to use for tunnels. + tunnelDir string + + // tunnel is the tunnel we're using. + tunnel tunnel.Tunnel + + // userAgent is the user agent we're using when communicating + // with the OONI backend (e.g., for the test helpers). + userAgent string +} + +// ErrEmptySoftwareName indicates the software name is empty. +var ErrEmptySoftwareName = errors.New("session: passed empty software name") + +// ErrEmptySoftwareVersion indicates the software version is empty. +var ErrEmptySoftwareVersion = errors.New("session: passed empty software version") + +// newState creates a new [state] instance. +func (s *Session) newState(ctx context.Context, req *Request) (*state, error) { + runtimex.Assert(req.Bootstrap != nil, "passed nil Bootstrap") + + if req.Bootstrap.SoftwareName == "" { + return nil, ErrEmptySoftwareName + } + + if req.Bootstrap.SoftwareVersion == "" { + return nil, ErrEmptySoftwareVersion + } + + logger := s.newLogger(req.Bootstrap.VerboseLogging) + + // Implementation note: the context we receive from the caller limits the + // whole lifetime of the tunnel we're going to create below. Because of + // that, we're not allowed to cancel this context or add a timeout to it. + + ts := newTickerService(ctx, s) + defer ts.stop() + + logger.Infof("creating key-value store at %s", req.Bootstrap.StateDir) + kvstore, err := kvstore.NewFS(req.Bootstrap.StateDir) + if err != nil { + logger.Warnf("cannot create key-value store: %s", err.Error()) + return nil, err + } + + logger.Infof("creating temporary directory inside %s", req.Bootstrap.TempDir) + if err := os.MkdirAll(req.Bootstrap.TempDir, 0700); err != nil { + logger.Warnf("cannot create temporary directory root: %s", err.Error()) + return nil, err + } + tempDir, err := os.MkdirTemp(req.Bootstrap.TempDir, "") + if err != nil { + logger.Warnf("cannot create session temporary directory: %s", err.Error()) + return nil, err + } + + logger.Infof("checking whether we need to create a circumvention tunnel") + tunnel, err := newTunnel(ctx, logger, req.Bootstrap) + if err != nil { + logger.Warnf("cannot create tunnel: %s", err.Error()) + return nil, err + } + + logger.Infof("creating a session byte counter") + counter := bytecounter.New() + + logger.Infof("creating a resolver for the session") + resolver := &sessionresolver.Resolver{ + ByteCounter: counter, + KVStore: kvstore, + Logger: logger, + ProxyURL: tunnel.SOCKS5ProxyURL(), // possibly nil, which is OK + } + + logger.Infof("creating an HTTP client for the session") + httpClient := sessionhttpclient.New(&sessionhttpclient.Config{ + ByteCounter: counter, + Logger: logger, + ProxyURL: tunnel.SOCKS5ProxyURL(), // possibly nil, which is OK + Resolver: resolver, + }) + + logger.Infof("creating the default user-agent string") + userAgent := fmt.Sprintf( + "%s/%s ooniprobe-engine/%s", + req.Bootstrap.SoftwareName, + req.Bootstrap.SoftwareVersion, + version.Version, + ) + + logger.Infof("creating a client to communicate with the OONI backend") + backendClient := backendclient.New(&backendclient.Config{ + BaseURL: nil, // use the default + KVStore: kvstore, + HTTPClient: httpClient, + Logger: logger, + UserAgent: userAgent, + }) + + logger.Infof("session bootstrap complete") + state := &state{ + backendClient: backendClient, + checkIn: nil, + counter: counter, + httpClient: httpClient, + kvstore: kvstore, + location: nil, + logger: logger, + resolver: resolver, + softwareName: req.Bootstrap.SoftwareName, + softwareVersion: req.Bootstrap.SoftwareVersion, + tempDir: tempDir, + torBinary: req.Bootstrap.TorBinary, + tunnelDir: req.Bootstrap.TunnelDir, + tunnel: tunnel, + userAgent: userAgent, + } + return state, nil +} + +// cleanup cleans the resources used by [state]. +func (s *state) cleanup() { + s.resolver.CloseIdleConnections() + s.httpClient.CloseIdleConnections() + s.tunnel.Stop() + os.RemoveAll(s.tempDir) +} diff --git a/internal/session/submit.go b/internal/session/submit.go new file mode 100644 index 0000000000..4696ea9337 --- /dev/null +++ b/internal/session/submit.go @@ -0,0 +1,40 @@ +package session + +import ( + "context" + + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// SubmitRequest is the request for the measurement submission API. +type SubmitRequest = model.Measurement + +// SubmitEvent is the result of submitting a measurement. +type SubmitEvent struct { + // Error is the geolocate result. + Error error +} + +// submit submits a measurement. +func (s *Session) submit(ctx context.Context, req *Request) { + s.emit(&Event{ + Submit: &SubmitEvent{ + Error: s.dosubmit(ctx, req), + }, + }) +} + +// dosubmit implements submit. +func (s *Session) dosubmit(ctx context.Context, req *Request) error { + runtimex.Assert(req.Submit != nil, "passed a nil Submit") + + if s.state == nil { + return ErrNotBootstrapped + } + + ts := newTickerService(ctx, s) + defer ts.stop() + + return s.state.backendClient.Submit(ctx, req.Submit) +} diff --git a/internal/session/tickerservice.go b/internal/session/tickerservice.go new file mode 100644 index 0000000000..978a6d744d --- /dev/null +++ b/internal/session/tickerservice.go @@ -0,0 +1,59 @@ +package session + +import ( + "context" + "time" +) + +// TickerEvent is an event emmitted by the [tickerService]. +type TickerEvent struct { + // ElapsedTime is the elapsed time since when the + // long running operation has been started. + ElapsedTime time.Duration +} + +// tickerServer is a ticker that emits a tick while a long running +// operation is alive. We use this ticker in the UI to make progress +// bars increase as well as to unblock the UI code and know when +// an operation has been running for too much time. +type tickerService struct { + cancel context.CancelFunc + sess *Session +} + +// newTickerService creates a [tickerService] running in the background +// and using the [Session] to emit [TickerEvents]. You should use the +// stop method of the [tickerService] to stop emitting events. +func newTickerService(ctx context.Context, sess *Session) *tickerService { + ctx, cancel := context.WithCancel(ctx) + ts := &tickerService{ + cancel: cancel, + sess: sess, + } + go ts.mainloop(ctx) + return ts +} + +// mainloop is the main loop of the [tickerService]. +func (ts *tickerService) mainloop(ctx context.Context) { + t0 := time.Now() + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case t := <-ticker.C: + ts.sess.emit(&Event{ + Ticker: &TickerEvent{ + ElapsedTime: t.Sub(t0), + }, + }) + case <-ctx.Done(): + return + } + } +} + +// stop stops the [tickerService] background goroutine. +func (ts *tickerService) stop() { + ts.cancel() +} diff --git a/internal/session/tunnel.go b/internal/session/tunnel.go new file mode 100644 index 0000000000..0ca0413107 --- /dev/null +++ b/internal/session/tunnel.go @@ -0,0 +1,118 @@ +package session + +import ( + "context" + "errors" + "fmt" + "net/url" + "time" + + "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/tunnel" +) + +// ErrUnsupportedTunnelScheme indicates we don't support the tunnel scheme. +var ErrUnsupportedTunnelScheme = errors.New("session: unsupported tunnel scheme") + +// newTunnel creates a new [tunnel.Tunnel] given a ProxyURL. +func newTunnel(ctx context.Context, logger model.Logger, req *BootstrapRequest) (tunnel.Tunnel, error) { + if req.ProxyURL == "" { + logger.Info("no need to create any tunnel") + return &nullTunnel{}, nil + } + URL, err := url.Parse(req.ProxyURL) + if err != nil { + return nil, err + } + switch scheme := URL.Scheme; scheme { + case "socks5": + logger.Infof("creating fake tunnel for %s", req.ProxyURL) + return &socks5Tunnel{URL}, nil + case "tor", "torsf": + logger.Infof("creating %s tunnel; please, be patient...", scheme) + return newTorOrTorsfTunnel(ctx, logger, req, scheme) + case "psiphon": + logger.Info("creating psiphon tunnel; please, be patient...") + return newPsiphonTunnel(ctx, logger, req) + default: + return nil, fmt.Errorf("%w: %s", ErrUnsupportedTunnelScheme, scheme) + } +} + +// nullTunnel is the absence of any tunnel. +type nullTunnel struct{} + +// BootstrapTime implements tunnel.Tunnel +func (t *nullTunnel) BootstrapTime() time.Duration { + return 0 +} + +// SOCKS5ProxyURL implements tunnel.Tunnel +func (t *nullTunnel) SOCKS5ProxyURL() *url.URL { + return nil // perfectly fine to return nil in this context +} + +// Stop implements tunnel.Tunnel +func (t *nullTunnel) Stop() { + // nothing +} + +// socks5Tunnel is a fake tunnel that returns a SOCKS5 URL. +type socks5Tunnel struct { + url *url.URL +} + +// BootstrapTime implements tunnel.Tunnel +func (t *socks5Tunnel) BootstrapTime() time.Duration { + return 0 +} + +// SOCKS5ProxyURL implements tunnel.Tunnel +func (t *socks5Tunnel) SOCKS5ProxyURL() *url.URL { + return t.url +} + +// Stop implements tunnel.Tunnel +func (t *socks5Tunnel) Stop() { + // nothing +} + +// newTorOrTorsfTunnel creates a tor or torsf tunnel. +func newTorOrTorsfTunnel( + ctx context.Context, + logger model.Logger, + req *BootstrapRequest, + scheme string, +) (tunnel.Tunnel, error) { + config := &tunnel.Config{ + Name: scheme, + Session: &engine.SessionTunnelEarlySession{}, + SnowflakeRendezvous: req.SnowflakeRendezvousMethod, + TunnelDir: req.TunnelDir, + Logger: logger, + TorArgs: req.TorArgs, + TorBinary: req.TorBinary, + } + tun, _, err := tunnel.Start(ctx, config) + return tun, err +} + +// newPsiphonTunnel creates a psiphon tunnel. +func newPsiphonTunnel( + ctx context.Context, + logger model.Logger, + req *BootstrapRequest, +) (tunnel.Tunnel, error) { + config := &tunnel.Config{ + Name: "psiphon", + Session: &engine.SessionTunnelEarlySession{}, + SnowflakeRendezvous: "", // not needed + TunnelDir: req.TunnelDir, + Logger: logger, + TorArgs: nil, // not needed + TorBinary: "", // not needed + } + tun, _, err := tunnel.Start(ctx, config) + return tun, err +} diff --git a/internal/session/webconnectivity.go b/internal/session/webconnectivity.go new file mode 100644 index 0000000000..2a7cf044ea --- /dev/null +++ b/internal/session/webconnectivity.go @@ -0,0 +1,85 @@ +package session + +import ( + "context" + "time" + + "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivitylte" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// WebConnectivityRequest is a request to run Web Connectivity. +type WebConnectivityRequest struct { + // Input is the URL to measure using Web Connectivity. + Input string + + // ReportID is the report ID to use. + ReportID string + + // TestStartTime is when we started running this test. + TestStartTime time.Time +} + +// WebConnectivityEvent is emitted after we have run +// a measurement using Web Connectivity. +type WebConnectivityEvent struct { + // Error indicates a fundamental error occurred + // when running this experiment. + Error error + + // Measurement is the measurement result. + Measurement *model.Measurement +} + +// webconnectivity performs a measurement using Web Connectivity. +func (s *Session) webconnectivity(ctx context.Context, req *Request) { + measurement, err := s.dowebconnectivity(ctx, req) + event := &Event{ + WebConnectivity: &WebConnectivityEvent{ + Error: err, + Measurement: measurement, + }, + } + s.emit(event) +} + +// dowebconnectivity implements webconnectivity. +func (s *Session) dowebconnectivity(ctx context.Context, req *Request) (*model.Measurement, error) { + runtimex.Assert(req.WebConnectivity != nil, "passed a nil WebConnectivity") + + if s.state == nil { + return nil, ErrNotBootstrapped + } + + ts := newTickerService(ctx, s) + defer ts.stop() + + adapter, err := newSessionAdapter(s.state) + if err != nil { + return nil, err + } + + cfg := &webconnectivitylte.Config{} + runner := webconnectivitylte.NewExperimentMeasurer(cfg) + measurement := model.NewMeasurement( + adapter.location, + runner.ExperimentName(), + runner.ExperimentVersion(), + req.WebConnectivity.TestStartTime, + req.WebConnectivity.ReportID, + s.state.softwareName, + s.state.softwareVersion, + req.WebConnectivity.Input, + ) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(model.DiscardLogger), + Measurement: measurement, + Session: adapter, + } + + if err := runner.Run(ctx, args); err != nil { + return nil, err + } + return measurement, nil +} diff --git a/internal/sessionresolver/resolver.go b/internal/sessionresolver/resolver.go index 0b0b124ae9..6181a05dc3 100644 --- a/internal/sessionresolver/resolver.go +++ b/internal/sessionresolver/resolver.go @@ -13,6 +13,7 @@ import ( "sync" "time" + "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/bytecounter" "github.com/ooni/probe-cli/v3/internal/measurexlite" "github.com/ooni/probe-cli/v3/internal/model" @@ -100,6 +101,7 @@ var ErrLookupHost = errors.New("sessionresolver: LookupHost failed") // multierror.Union error on failure, so you can see individual errors // and get a better picture of what's been going wrong. func (r *Resolver) LookupHost(ctx context.Context, hostname string) ([]string, error) { + log.Warnf("PROXY URL: %+v", r.ProxyURL) state := r.readstatedefault() r.maybeConfusion(state, time.Now().UnixNano()) defer r.writestate(state) From 1afc71f00d1a5f981a0b53b8dc2502846423e025 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 8 Feb 2023 16:34:51 +0000 Subject: [PATCH 13/17] better code --- .../{session => cmd/dismantle2}/client.go | 27 +++-- internal/cmd/dismantle2/main.go | 2 +- internal/model/optional.go | 33 ++++++ internal/session/bootstrap.go | 27 +++-- internal/session/checkin.go | 22 ++-- internal/session/doc.go | 5 + internal/session/geolocate.go | 32 ++--- internal/session/logger.go | 18 +-- internal/session/progressbar.go | 19 ++- internal/session/session.go | 105 +++++++++------- internal/session/sessionadapter.go | 16 ++- internal/session/state.go | 112 ++++++++++++------ internal/session/submit.go | 16 ++- internal/session/tickerservice.go | 2 +- internal/session/tunnel.go | 4 + internal/session/webconnectivity.go | 30 +++-- 16 files changed, 307 insertions(+), 163 deletions(-) rename internal/{session => cmd/dismantle2}/client.go (77%) create mode 100644 internal/model/optional.go create mode 100644 internal/session/doc.go diff --git a/internal/session/client.go b/internal/cmd/dismantle2/client.go similarity index 77% rename from internal/session/client.go rename to internal/cmd/dismantle2/client.go index 3c3ba10a1d..422428a3d2 100644 --- a/internal/session/client.go +++ b/internal/cmd/dismantle2/client.go @@ -1,4 +1,4 @@ -package session +package main import ( "context" @@ -6,6 +6,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/geolocate" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/session" ) // Client is a client for a [Session]. You should use a [Client] @@ -13,7 +14,7 @@ import ( type Client struct { logger model.Logger once sync.Once - session *Session + session *session.Session } // NewClient creates a [Client] instance. @@ -21,13 +22,13 @@ func NewClient(logger model.Logger) *Client { return &Client{ logger: logger, once: sync.Once{}, - session: New(), + session: session.New(), } } // Bootstrap bootstraps a session. -func (c *Client) Bootstrap(ctx context.Context, req *BootstrapRequest) error { - if err := c.session.Send(ctx, &Request{Bootstrap: req}); err != nil { +func (c *Client) Bootstrap(ctx context.Context, req *session.BootstrapRequest) error { + if err := c.session.Send(ctx, &session.Request{Bootstrap: req}); err != nil { return err } for { @@ -51,8 +52,8 @@ func (c *Client) Bootstrap(ctx context.Context, req *BootstrapRequest) error { } // Geolocate performs geolocation. You must run bootstrap first. -func (c *Client) Geolocate(ctx context.Context, req *GeolocateRequest) (*geolocate.Results, error) { - if err := c.session.Send(ctx, &Request{Geolocate: req}); err != nil { +func (c *Client) Geolocate(ctx context.Context, req *session.GeolocateRequest) (*geolocate.Results, error) { + if err := c.session.Send(ctx, &session.Request{Geolocate: req}); err != nil { return nil, err } for { @@ -76,8 +77,8 @@ func (c *Client) Geolocate(ctx context.Context, req *GeolocateRequest) (*geoloca } // CheckIn calls the check-in API. You must run bootstrap first. -func (c *Client) CheckIn(ctx context.Context, req *CheckInRequest) (*model.OOAPICheckInResult, error) { - if err := c.session.Send(ctx, &Request{CheckIn: req}); err != nil { +func (c *Client) CheckIn(ctx context.Context, req *session.CheckInRequest) (*model.OOAPICheckInResult, error) { + if err := c.session.Send(ctx, &session.Request{CheckIn: req}); err != nil { return nil, err } for { @@ -102,8 +103,8 @@ func (c *Client) CheckIn(ctx context.Context, req *CheckInRequest) (*model.OOAPI // WebConnectivity runs a single-URL Web Connectivity measurement. func (c *Client) WebConnectivity( - ctx context.Context, req *WebConnectivityRequest) (*model.Measurement, error) { - if err := c.session.Send(ctx, &Request{WebConnectivity: req}); err != nil { + ctx context.Context, req *session.WebConnectivityRequest) (*model.Measurement, error) { + if err := c.session.Send(ctx, &session.Request{WebConnectivity: req}); err != nil { return nil, err } for { @@ -128,7 +129,7 @@ func (c *Client) WebConnectivity( // Submit submits a measurement. func (c *Client) Submit(ctx context.Context, measurement *model.Measurement) error { - if err := c.session.Send(ctx, &Request{Submit: measurement}); err != nil { + if err := c.session.Send(ctx, &session.Request{Submit: measurement}); err != nil { return err } for { @@ -160,7 +161,7 @@ func (c *Client) Close() (err error) { } // emitLog emits a log event. -func (c *Client) emitLog(ev *LogEvent) { +func (c *Client) emitLog(ev *session.LogEvent) { switch ev.Level { case "DEBUG": c.logger.Debug(ev.Message) diff --git a/internal/cmd/dismantle2/main.go b/internal/cmd/dismantle2/main.go index 3d91bef5fc..2c588664c3 100644 --- a/internal/cmd/dismantle2/main.go +++ b/internal/cmd/dismantle2/main.go @@ -19,7 +19,7 @@ const softwareVersion = "0.1.0-dev" func main() { log.SetLevel(log.DebugLevel) - client := session.NewClient(log.Log) + client := NewClient(log.Log) ctx := context.Background() bootstrapRequest := &session.BootstrapRequest{ diff --git a/internal/model/optional.go b/internal/model/optional.go new file mode 100644 index 0000000000..1eb2cf6410 --- /dev/null +++ b/internal/model/optional.go @@ -0,0 +1,33 @@ +package model + +import ( + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// OptionalPtr is an optional pointer. +type OptionalPtr[Type any] struct { + v *Type +} + +// NewOptionalPtr creates a new OptionalPtr. +func NewOptionalPtr[Type any](v *Type) OptionalPtr[Type] { + runtimex.Assert(v != nil, "passed nil pointer") + return OptionalPtr[Type]{v} +} + +// Unwrap returns the underlying pointer. This function panics +// if the underlying ptr is nil. +func (op OptionalPtr[Type]) Unwrap() *Type { + runtimex.Assert(op.IsSome(), "not initialized") + return op.v +} + +// IsSome returns whether the underlying ptr is not nil. +func (op OptionalPtr[Type]) IsSome() bool { + return op.v != nil +} + +// IsNone returns whether the underlying ptr is nil. +func (op OptionalPtr[Type]) IsNone() bool { + return !op.IsSome() +} diff --git a/internal/session/bootstrap.go b/internal/session/bootstrap.go index a0567b459b..f7b2243d48 100644 --- a/internal/session/bootstrap.go +++ b/internal/session/bootstrap.go @@ -1,12 +1,19 @@ package session +// +// Bootstrapping a measurement session. +// + import ( "context" "errors" + + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" ) // BootstrapRequest is a request to bootstrap the [Session] and -// contains the arguments requested by the bootstrap. You can +// contains the arguments required by the bootstrap. You can // boostrap a [Session] just once. All operations you would like // to perform with a [Session] require a boostrap first. type BootstrapRequest struct { @@ -40,11 +47,14 @@ type BootstrapRequest struct { SoftwareVersion string // TorArgs OPTIONALLY passes command line arguments to tor - // when the ProxyURL scheme is "tor" or "torsf". + // when the ProxyURL scheme is "tor" or "torsf". We will only + // use these arguments for bootstrapping, not for measuring. TorArgs []string // TorBinary OPTIONALLY tells the engine to use a specific - // binary for starting the "tor" and "torsf" tunnels. + // binary for starting the "tor" and "torsf" tunnels. If this + // argument is set, we will also use it for measuring for + // each experiment that requires tor. TorBinary string // TempDir is the MANDATORY base directory in which @@ -66,8 +76,9 @@ type BootstrapEvent struct { } // boostrap bootstraps a session. -func (s *Session) bootstrap(ctx context.Context, req *Request) { - s.emit(&Event{ +func (s *Session) bootstrap(ctx context.Context, req *BootstrapRequest) { + runtimex.Assert(req != nil, "passed nil req") + s.maybeEmit(&Event{ Bootstrap: &BootstrapEvent{ Error: s.dobootstrap(ctx, req), }, @@ -78,14 +89,14 @@ func (s *Session) bootstrap(ctx context.Context, req *Request) { var ErrAlreadyBootstrapped = errors.New("session: already bootstrapped") // dobootstrap implements bootstrap. -func (s *Session) dobootstrap(ctx context.Context, req *Request) error { - if s.state != nil { +func (s *Session) dobootstrap(ctx context.Context, req *BootstrapRequest) error { + if s.state.IsSome() { return ErrAlreadyBootstrapped } state, err := s.newState(ctx, req) if err != nil { return err } - s.state = state + s.state = model.NewOptionalPtr(state) return nil } diff --git a/internal/session/checkin.go b/internal/session/checkin.go index 019e687761..b2bb02e882 100644 --- a/internal/session/checkin.go +++ b/internal/session/checkin.go @@ -1,5 +1,9 @@ package session +// +// Code to call the check-in API. +// + import ( "context" @@ -20,32 +24,30 @@ type CheckInEvent struct { } // checkin calls the check-in API. -func (s *Session) checkin(ctx context.Context, req *Request) { +func (s *Session) checkin(ctx context.Context, req *CheckInRequest) { + runtimex.Assert(req != nil, "passed a nil req") result, err := s.docheckin(ctx, req) - event := &Event{ + s.maybeEmit(&Event{ CheckIn: &CheckInEvent{ Error: err, Result: result, }, - } - s.emit(event) + }) } // docheckin implements checkin. -func (s *Session) docheckin(ctx context.Context, req *Request) (*model.OOAPICheckInResult, error) { - runtimex.Assert(req.CheckIn != nil, "passed a nil CheckIn") - - if s.state == nil { +func (s *Session) docheckin(ctx context.Context, req *CheckInRequest) (*model.OOAPICheckInResult, error) { + if s.state.IsNone() { return nil, ErrNotBootstrapped } ts := newTickerService(ctx, s) defer ts.stop() - result, err := s.state.backendClient.CheckIn(ctx, req.CheckIn) + result, err := s.state.Unwrap().backendClient.CheckIn(ctx, req) if err != nil { return nil, err } - s.state.checkIn = result + s.state.Unwrap().checkIn = model.NewOptionalPtr(result) return result, nil } diff --git a/internal/session/doc.go b/internal/session/doc.go new file mode 100644 index 0000000000..9f37164713 --- /dev/null +++ b/internal/session/doc.go @@ -0,0 +1,5 @@ +// Package session implements a measurement session. The design of +// this package is such that we can split the measurement engine proper +// and the application using it. In particular, this design is such +// that it would be easy to expose this API as a C library. +package session diff --git a/internal/session/geolocate.go b/internal/session/geolocate.go index cbc606cb1f..49a7028d21 100644 --- a/internal/session/geolocate.go +++ b/internal/session/geolocate.go @@ -1,5 +1,9 @@ package session +// +// Geolocating a probe. +// + import ( "context" "errors" @@ -17,30 +21,28 @@ type GeolocateEvent struct { // Error is the geolocate result. Error error - // Location is the probe location. + // Location is the geolocated location. Location *geolocate.Results } // geolocate performs a geolocation. -func (s *Session) geolocate(ctx context.Context, req *Request) { +func (s *Session) geolocate(ctx context.Context, req *GeolocateRequest) { + runtimex.Assert(req != nil, "passed a nil req") location, err := s.dogeolocate(ctx, req) - event := &Event{ + s.maybeEmit(&Event{ Geolocate: &GeolocateEvent{ Error: err, Location: location, }, - } - s.emit(event) + }) } -// ErrNotBootstrapped indicates we didn't bootstrap a session. +// ErrNotBootstrapped indicates we have not bootstrapped the session yet. var ErrNotBootstrapped = errors.New("session: not bootstrapped") // dogeolocate implements geolocate. -func (s *Session) dogeolocate(ctx context.Context, req *Request) (*geolocate.Results, error) { - runtimex.Assert(req.Geolocate != nil, "passed a nil Geolocate") - - if s.state == nil { +func (s *Session) dogeolocate(ctx context.Context, req *GeolocateRequest) (*geolocate.Results, error) { + if s.state.IsNone() { return nil, ErrNotBootstrapped } @@ -48,11 +50,11 @@ func (s *Session) dogeolocate(ctx context.Context, req *Request) (*geolocate.Res defer ts.stop() geolocateConfig := geolocate.Config{ - Resolver: s.state.resolver, - Logger: s.state.logger, - UserAgent: model.HTTPHeaderUserAgent, + Resolver: s.state.Unwrap().resolver, + Logger: s.state.Unwrap().logger, + UserAgent: model.HTTPHeaderUserAgent, // do not disclose we are OONI } - task := geolocate.NewTask(geolocateConfig) // XXX + task := geolocate.NewTask(geolocateConfig) // TODO(bassosimone): make this a pointer. // TODO(bassosimone): we should make geolocate.Results a type // in the internal/model package and use the ~typical naming for @@ -62,6 +64,6 @@ func (s *Session) dogeolocate(ctx context.Context, req *Request) (*geolocate.Res return nil, err } - s.state.location = location + s.state.Unwrap().location = model.NewOptionalPtr(location) return location, nil } diff --git a/internal/session/logger.go b/internal/session/logger.go index 8cb970c326..067dbcfe0c 100644 --- a/internal/session/logger.go +++ b/internal/session/logger.go @@ -1,5 +1,9 @@ package session +// +// A model.Logger emitting events on a Session output channel +// + import ( "fmt" "time" @@ -7,7 +11,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/model" ) -// LogEvent is an event emitted by a logger. +// LogEvent is a log event. type LogEvent struct { // Timestamp is the log timestamp. Timestamp time.Time @@ -37,7 +41,7 @@ type sessionLogger struct { // Debug implements model.Logger func (sl *sessionLogger) Debug(msg string) { if sl.verbose { - sl.emit("DEBUG", msg) + sl.maybeEmit("DEBUG", msg) } } @@ -50,7 +54,7 @@ func (sl *sessionLogger) Debugf(format string, v ...interface{}) { // Info implements model.Logger func (sl *sessionLogger) Info(msg string) { - sl.emit("INFO", msg) + sl.maybeEmit("INFO", msg) } // Infof implements model.Logger @@ -60,7 +64,7 @@ func (sl *sessionLogger) Infof(format string, v ...interface{}) { // Warn implements model.Logger func (sl *sessionLogger) Warn(msg string) { - sl.emit("WARNING", msg) + sl.maybeEmit("WARNING", msg) } // Warnf implements model.Logger @@ -68,8 +72,8 @@ func (sl *sessionLogger) Warnf(format string, v ...interface{}) { sl.Warn(fmt.Sprintf(format, v...)) } -// emit emits a log message. -func (sl *sessionLogger) emit(level, message string) { +// maybeEmit emits a log message if the output channel buffer is not full. +func (sl *sessionLogger) maybeEmit(level, message string) { ev := &Event{ Log: &LogEvent{ Timestamp: time.Now(), @@ -77,5 +81,5 @@ func (sl *sessionLogger) emit(level, message string) { Message: message, }, } - sl.session.emit(ev) + sl.session.maybeEmit(ev) } diff --git a/internal/session/progressbar.go b/internal/session/progressbar.go index 828e41761b..7928c484b2 100644 --- a/internal/session/progressbar.go +++ b/internal/session/progressbar.go @@ -1,17 +1,24 @@ package session +// +// Progress bar for experiments that manage their own progress +// bar such as DASH, NDT, HIRL, HHFM. +// + import ( "time" "github.com/ooni/probe-cli/v3/internal/model" ) -// ProgressEvent indicates the progress in a task completion. +// ProgressEvent indicates the progress in a task completion +// for tasks that manage their own progress bar. type ProgressEvent struct { // Timestamp is the timestamp. Timestamp time.Time - // Completion is a number between 0 and 1. + // Completion is a number between 0 and 1 indicating + // how close we are to completion. Completion float64 // Message is the message. @@ -27,11 +34,11 @@ var _ model.ExperimentCallbacks = &progressBar{} // OnProgress implements model.ExperimentCallbacks func (pb *progressBar) OnProgress(completion float64, message string) { - pb.emit(completion, message) + pb.maybeEmit(completion, message) } -// emit emits a progress event. -func (pb *progressBar) emit(completion float64, message string) { +// emit emits a progress event unless the output channel is full. +func (pb *progressBar) maybeEmit(completion float64, message string) { ev := &Event{ Progress: &ProgressEvent{ Timestamp: time.Now(), @@ -39,5 +46,5 @@ func (pb *progressBar) emit(completion float64, message string) { Message: message, }, } - pb.session.emit(ev) + pb.session.maybeEmit(ev) } diff --git a/internal/session/session.go b/internal/session/session.go index 502589abb9..8dccce4868 100644 --- a/internal/session/session.go +++ b/internal/session/session.go @@ -1,40 +1,52 @@ -// Package session implements a measurement session. The design of -// this package is such that we can split the measurement engine proper -// and the application using it. In particular, this design is such -// that it would be easy to expose this API as a C library. -// -// The general usage of this package is the following: +package session + // -// 1. XXX document +// Public definition of Session // -// Go packages should not use this package directly but rather use -// the sessionclient package, which provides idiomatic wrappers. -package session import ( "context" "errors" "log" "sync" + + "github.com/ooni/probe-cli/v3/internal/model" ) -// Session is a measurement session. +// Session is a measurement session. The zero value of this structure +// is invalid. You must use the [New] factory to create a new valid instance. +// +// A session consists of a background goroutine to which you Send +// [Request]s. Each [Request] causes the background goroutine to +// start running a long-running task. You should Recv [Event]s +// emitted by the task until you find the matching [Event] that implies +// that the task you started has finished running. +// +// While running, a task emits "ticker" events that you can use to +// fill a progress bar and to decide when the task should stop running. To +// stop a long-running task, you call [Close], which forces the background +// goroutine to stop as soon as possible. +// +// Once a [Session] has been terminated using [Close] you loose all +// the [Session] state and you must create a new [Session]. type Session struct { // cancel allows us to terminate the backround goroutine. cancel context.CancelFunc - // input is the channel to send input to the background goroutine. + // input is the channel to send [Request] to the background goroutine. input chan *Request - // once allows us to run cleanups just once. + // once allows us to cleanup the state just once. once sync.Once - // output is the channel from which we read the emitted events. + // output is the channel from which we read the emitted [Event]s. output chan *Event // state is the background goroutine's state, which only - // the background goroutine is allowed to modify. - state *state + // the background goroutine is allowed to modify. We start + // with a nil state and create state using the bootstrap + // long running task. + state model.OptionalPtr[state] // terminated is closed when the background goroutine terminates. terminated chan any @@ -44,12 +56,14 @@ type Session struct { // a background goroutine that will handle incoming [Request]s. func New() *Session { ctx, cancel := context.WithCancel(context.Background()) + // Implementation note: we use buffered channels for input and + // output to avoid loosing events in _most_ cases. s := &Session{ cancel: cancel, input: make(chan *Request, 1024), once: sync.Once{}, output: make(chan *Event, 1024), - state: nil, + state: model.OptionalPtr[state]{}, terminated: make(chan any), } go s.mainloop(ctx) @@ -59,29 +73,32 @@ func New() *Session { // Request requests a [Session] to perform a background task. This // struct contains several pointers. Each of them indicates a specific // task the [Session] should run. The [Session] will go through each -// pointer and only consider the first one that is not nil. +// pointer and only consider the first one that is not nil. As such +// it's pointless to initialize more than one pointer. type Request struct { // Bootstrap indicates that the [Session] should bootstrap // and contains bootstrap configuration. Bootstrap *BootstrapRequest - // CheckIn indicates that the [Session] should call the check-in API. + // CheckIn indicates that the [Session] should call the check-in API + // and only works if you have already bootstrapped. CheckIn *CheckInRequest // Geolocate indicates that the [Session] should obtain - // the current probe's geolocation. + // the current probe's geolocation. This task requires you + // to successfully bootstrap a session first. Geolocate *GeolocateRequest - // Submit indicates that the [Session] should submit a measurement. + // Submit indicates that the [Session] should submit a + // measurement. You must bootstrap first. Submit *SubmitRequest // WebConnectivity indicates that the [Session] should - // run the Web Connectivity experiment. + // run the Web Connectivity experiment. You must bootstrap first. WebConnectivity *WebConnectivityRequest } -// ErrSessionTerminated indicates that the background goroutine -// servicing the [Session] [Request]s has stopped. +// ErrSessionTerminated indicates that the background goroutine has terminated. var ErrSessionTerminated = errors.New("session: terminated") // Send sends a [Request] to a [Session]. This function will return an @@ -97,7 +114,7 @@ func (s *Session) Send(ctx context.Context, req *Request) error { } } -// Close stops the goroutine running in the background and release +// Close stops the goroutine running in the background and releases // all the resources allocated by the [Session]. func (s *Session) Close() error { s.once.Do(s.joincleanup) @@ -108,8 +125,9 @@ func (s *Session) Close() error { func (s *Session) joincleanup() { s.cancel() <-s.terminated - if s.state != nil { - s.state.cleanup() + if s.state.IsSome() { + s.state.Unwrap().cleanup() + s.state = model.OptionalPtr[state]{} // just to be tidy } } @@ -127,8 +145,10 @@ func (s *Session) Recv(ctx context.Context) (*Event, error) { } } -// emit emits an [Event]. -func (s *Session) emit(ev *Event) { +// maybeEmit emits an [Event] if possible. If the output channel +// buffer is full, we are not going to emit the event. In such +// a case, we will print a log message to the standard error file. +func (s *Session) maybeEmit(ev *Event) { select { case s.output <- ev: default: @@ -136,7 +156,9 @@ func (s *Session) emit(ev *Event) { } } -// Event is an event emitted by a [Session]. +// Event is an event emitted by a [Session]. Only one of the pointers +// of the [Event] will be set. The pointer being set uniquely identifies +// the specific event that has occurred. type Event struct { // Bootstrap is emitted at the end of the bootstrap. Bootstrap *BootstrapEvent @@ -147,19 +169,23 @@ type Event struct { // Geolocate is emitted at the end of geolocate. Geolocate *GeolocateEvent - // Log is a log event. + // Log is a log event. Any task will emit log events. Log *LogEvent - // Progress is a progress event. + // Progress is a progress event. Only experiments that print + // their own progress will emit this event. Progress *ProgressEvent // Submit is emitted after a measurement submission. Submit *SubmitEvent - // Ticker is a ticker event. + // Ticker is a ticker event. Any task will emit ticker + // events so that you can increase a progress bar and + // decide whether the task should be stopped. Ticker *TickerEvent - // WebConnectivity is the Web Connectivity event. + // WebConnectivity is the Web Connectivity event, emitted + // once we finished measuring a given URL. WebConnectivity *WebConnectivityEvent } @@ -178,25 +204,24 @@ func (s *Session) mainloop(ctx context.Context) { // handle handles an incoming [Request]. func (s *Session) handle(ctx context.Context, req *Request) { - // TODO(bassosimone): rewrite trying to avoid all these ifs. if req.Bootstrap != nil { - s.bootstrap(ctx, req) + s.bootstrap(ctx, req.Bootstrap) return } if req.CheckIn != nil { - s.checkin(ctx, req) + s.checkin(ctx, req.CheckIn) return } if req.Geolocate != nil { - s.geolocate(ctx, req) + s.geolocate(ctx, req.Geolocate) return } if req.Submit != nil { - s.submit(ctx, req) + s.submit(ctx, req.Submit) return } if req.WebConnectivity != nil { - s.webconnectivity(ctx, req) + s.webconnectivity(ctx, req.WebConnectivity) return } } diff --git a/internal/session/sessionadapter.go b/internal/session/sessionadapter.go index afa01de8ee..58eccea016 100644 --- a/internal/session/sessionadapter.go +++ b/internal/session/sessionadapter.go @@ -1,5 +1,9 @@ package session +// +// Adapter to pass model.ExperimentSession to experiments +// + import ( "context" "errors" @@ -28,18 +32,18 @@ var ErrNoCheckIn = errors.New("session: no check-in information") // newSessionAdapter creates a new [sessionAdapter] instance. func newSessionAdapter(state *state) (*sessionAdapter, error) { - if state.location == nil { + if state.location.IsNone() { return nil, ErrNoLocation } - if state.checkIn == nil { + if state.checkIn.IsNone() { return nil, ErrNoCheckIn } sa := &sessionAdapter{ httpClient: state.httpClient, - location: state.location, + location: state.location.Unwrap(), logger: state.logger, tempDir: state.tempDir, - testHelpers: state.checkIn.Conf.TestHelpers, + testHelpers: state.checkIn.Unwrap().Conf.TestHelpers, torBinary: state.torBinary, tunnelDir: state.tunnelDir, userAgent: state.userAgent, @@ -92,7 +96,9 @@ func (es *sessionAdapter) TempDir() string { // TorArgs implements model.ExperimentSession func (es *sessionAdapter) TorArgs() []string { - return []string{} // TODO(bassosimone): this field is only meaningful for bootstrap + // TODO(bassosimone): this field is only meaningful for bootstrap. So, it is + // wrong that we include it into the ExperimentSession. + return []string{} } // TorBinary implements model.ExperimentSession diff --git a/internal/session/state.go b/internal/session/state.go index df63ad7bf3..a8bac2e63e 100644 --- a/internal/session/state.go +++ b/internal/session/state.go @@ -1,5 +1,9 @@ package session +// +// State of a bootstrapped session. +// + import ( "context" "errors" @@ -18,14 +22,14 @@ import ( "github.com/ooni/probe-cli/v3/internal/version" ) -// state is the session's state. Only the background -// goroutine is allowed to manipulate it. +// state is the boostrapped [Session] state. Only the background +// goroutine is allowed to manipulate the [state]. type state struct { // backendClient is the backend client we're using. backendClient *backendclient.Client - // checkIn is the most recently fetched check-in result or nil. - checkIn *model.OOAPICheckInResult + // checkIn is the most recently fetched check-in result. + checkIn model.OptionalPtr[model.OOAPICheckInResult] // counter is the bytecounter we're using. counter *bytecounter.Counter @@ -36,8 +40,8 @@ type state struct { // kvstore is the session's key-value store. kvstore model.KeyValueStore - // location is the most recently resolved location or nil. - location *geolocate.Results + // location is the most recently resolved location. + location model.OptionalPtr[geolocate.Results] // logger is the model.Logger we're using. logger model.Logger @@ -72,6 +76,14 @@ type state struct { userAgent string } +// cleanup cleans the resources used by [state]. +func (s *state) cleanup() { + s.resolver.CloseIdleConnections() + s.httpClient.CloseIdleConnections() + s.tunnel.Stop() + os.RemoveAll(s.tempDir) +} + // ErrEmptySoftwareName indicates the software name is empty. var ErrEmptySoftwareName = errors.New("session: passed empty software name") @@ -79,18 +91,15 @@ var ErrEmptySoftwareName = errors.New("session: passed empty software name") var ErrEmptySoftwareVersion = errors.New("session: passed empty software version") // newState creates a new [state] instance. -func (s *Session) newState(ctx context.Context, req *Request) (*state, error) { - runtimex.Assert(req.Bootstrap != nil, "passed nil Bootstrap") - - if req.Bootstrap.SoftwareName == "" { +func (s *Session) newState(ctx context.Context, req *BootstrapRequest) (*state, error) { + if req.SoftwareName == "" { return nil, ErrEmptySoftwareName } - - if req.Bootstrap.SoftwareVersion == "" { + if req.SoftwareVersion == "" { return nil, ErrEmptySoftwareVersion } - logger := s.newLogger(req.Bootstrap.VerboseLogging) + logger := s.newLogger(req.VerboseLogging) // Implementation note: the context we receive from the caller limits the // whole lifetime of the tunnel we're going to create below. Because of @@ -99,31 +108,51 @@ func (s *Session) newState(ctx context.Context, req *Request) (*state, error) { ts := newTickerService(ctx, s) defer ts.stop() - logger.Infof("creating key-value store at %s", req.Bootstrap.StateDir) - kvstore, err := kvstore.NewFS(req.Bootstrap.StateDir) + logger.Infof("creating key-value store at %s", req.StateDir) + kvstore, err := kvstore.NewFS(req.StateDir) if err != nil { logger.Warnf("cannot create key-value store: %s", err.Error()) return nil, err } - logger.Infof("creating temporary directory inside %s", req.Bootstrap.TempDir) - if err := os.MkdirAll(req.Bootstrap.TempDir, 0700); err != nil { - logger.Warnf("cannot create temporary directory root: %s", err.Error()) - return nil, err - } - tempDir, err := os.MkdirTemp(req.Bootstrap.TempDir, "") + tempDir, err := stateNewTempDir(logger, req) if err != nil { - logger.Warnf("cannot create session temporary directory: %s", err.Error()) + // warning message already printed return nil, err } logger.Infof("checking whether we need to create a circumvention tunnel") - tunnel, err := newTunnel(ctx, logger, req.Bootstrap) + tunnel, err := newTunnel(ctx, logger, req) if err != nil { logger.Warnf("cannot create tunnel: %s", err.Error()) return nil, err } + state := newStateCannotFail( + logger, + kvstore, + tunnel, + tempDir, + req, + ) + return state, nil +} + +// newStateCannotFail constructs a [state] once we have +// performed all operations that may fail. +func newStateCannotFail( + logger model.Logger, + kvstore model.KeyValueStore, + tunnel tunnel.Tunnel, + tempDir string, + req *BootstrapRequest, +) *state { + runtimex.Assert(logger != nil, "passed a nil logger") + runtimex.Assert(kvstore != nil, "passed a nil kvstore") + runtimex.Assert(tunnel != nil, "passed a nil tunnel") + runtimex.Assert(tempDir != "", "passed an empty tempDir") + runtimex.Assert(req != nil, "passed a nil req") + logger.Infof("creating a session byte counter") counter := bytecounter.New() @@ -146,8 +175,8 @@ func (s *Session) newState(ctx context.Context, req *Request) (*state, error) { logger.Infof("creating the default user-agent string") userAgent := fmt.Sprintf( "%s/%s ooniprobe-engine/%s", - req.Bootstrap.SoftwareName, - req.Bootstrap.SoftwareVersion, + req.SoftwareName, + req.SoftwareVersion, version.Version, ) @@ -163,28 +192,35 @@ func (s *Session) newState(ctx context.Context, req *Request) (*state, error) { logger.Infof("session bootstrap complete") state := &state{ backendClient: backendClient, - checkIn: nil, + checkIn: model.OptionalPtr[model.OOAPICheckInResult]{}, counter: counter, httpClient: httpClient, kvstore: kvstore, - location: nil, + location: model.OptionalPtr[geolocate.Results]{}, logger: logger, resolver: resolver, - softwareName: req.Bootstrap.SoftwareName, - softwareVersion: req.Bootstrap.SoftwareVersion, + softwareName: req.SoftwareName, + softwareVersion: req.SoftwareVersion, tempDir: tempDir, - torBinary: req.Bootstrap.TorBinary, - tunnelDir: req.Bootstrap.TunnelDir, + torBinary: req.TorBinary, + tunnelDir: req.TunnelDir, tunnel: tunnel, userAgent: userAgent, } - return state, nil + return state } -// cleanup cleans the resources used by [state]. -func (s *state) cleanup() { - s.resolver.CloseIdleConnections() - s.httpClient.CloseIdleConnections() - s.tunnel.Stop() - os.RemoveAll(s.tempDir) +// stateNewTempDir creates a new temporary directory for [state]. +func stateNewTempDir(logger model.Logger, req *BootstrapRequest) (string, error) { + logger.Infof("creating temporary directory inside %s", req.TempDir) + if err := os.MkdirAll(req.TempDir, 0700); err != nil { + logger.Warnf("cannot create temporary directory root: %s", err.Error()) + return "", err + } + tempDir, err := os.MkdirTemp(req.TempDir, "") + if err != nil { + logger.Warnf("cannot create session temporary directory: %s", err.Error()) + return "", err + } + return tempDir, nil } diff --git a/internal/session/submit.go b/internal/session/submit.go index 4696ea9337..cb7bc2b84b 100644 --- a/internal/session/submit.go +++ b/internal/session/submit.go @@ -1,5 +1,9 @@ package session +// +// Submitting measurements +// + import ( "context" @@ -17,8 +21,8 @@ type SubmitEvent struct { } // submit submits a measurement. -func (s *Session) submit(ctx context.Context, req *Request) { - s.emit(&Event{ +func (s *Session) submit(ctx context.Context, req *SubmitRequest) { + s.maybeEmit(&Event{ Submit: &SubmitEvent{ Error: s.dosubmit(ctx, req), }, @@ -26,15 +30,15 @@ func (s *Session) submit(ctx context.Context, req *Request) { } // dosubmit implements submit. -func (s *Session) dosubmit(ctx context.Context, req *Request) error { - runtimex.Assert(req.Submit != nil, "passed a nil Submit") +func (s *Session) dosubmit(ctx context.Context, req *SubmitRequest) error { + runtimex.Assert(req != nil, "passed a nil req") - if s.state == nil { + if s.state.IsNone() { return ErrNotBootstrapped } ts := newTickerService(ctx, s) defer ts.stop() - return s.state.backendClient.Submit(ctx, req.Submit) + return s.state.Unwrap().backendClient.Submit(ctx, req) } diff --git a/internal/session/tickerservice.go b/internal/session/tickerservice.go index 978a6d744d..22b5e57327 100644 --- a/internal/session/tickerservice.go +++ b/internal/session/tickerservice.go @@ -42,7 +42,7 @@ func (ts *tickerService) mainloop(ctx context.Context) { for { select { case t := <-ticker.C: - ts.sess.emit(&Event{ + ts.sess.maybeEmit(&Event{ Ticker: &TickerEvent{ ElapsedTime: t.Sub(t0), }, diff --git a/internal/session/tunnel.go b/internal/session/tunnel.go index 0ca0413107..953649dfb8 100644 --- a/internal/session/tunnel.go +++ b/internal/session/tunnel.go @@ -1,5 +1,9 @@ package session +// +// Creating tunnels +// + import ( "context" "errors" diff --git a/internal/session/webconnectivity.go b/internal/session/webconnectivity.go index 2a7cf044ea..9760eb745a 100644 --- a/internal/session/webconnectivity.go +++ b/internal/session/webconnectivity.go @@ -1,5 +1,9 @@ package session +// +// Running the Web Connectivity experiment +// + import ( "context" "time" @@ -33,29 +37,29 @@ type WebConnectivityEvent struct { } // webconnectivity performs a measurement using Web Connectivity. -func (s *Session) webconnectivity(ctx context.Context, req *Request) { +func (s *Session) webconnectivity(ctx context.Context, req *WebConnectivityRequest) { + runtimex.Assert(req != nil, "passed a nil req") measurement, err := s.dowebconnectivity(ctx, req) - event := &Event{ + s.maybeEmit(&Event{ WebConnectivity: &WebConnectivityEvent{ Error: err, Measurement: measurement, }, - } - s.emit(event) + }) } // dowebconnectivity implements webconnectivity. -func (s *Session) dowebconnectivity(ctx context.Context, req *Request) (*model.Measurement, error) { - runtimex.Assert(req.WebConnectivity != nil, "passed a nil WebConnectivity") +func (s *Session) dowebconnectivity( + ctx context.Context, req *WebConnectivityRequest) (*model.Measurement, error) { - if s.state == nil { + if s.state.IsNone() { return nil, ErrNotBootstrapped } ts := newTickerService(ctx, s) defer ts.stop() - adapter, err := newSessionAdapter(s.state) + adapter, err := newSessionAdapter(s.state.Unwrap()) if err != nil { return nil, err } @@ -66,11 +70,11 @@ func (s *Session) dowebconnectivity(ctx context.Context, req *Request) (*model.M adapter.location, runner.ExperimentName(), runner.ExperimentVersion(), - req.WebConnectivity.TestStartTime, - req.WebConnectivity.ReportID, - s.state.softwareName, - s.state.softwareVersion, - req.WebConnectivity.Input, + req.TestStartTime, + req.ReportID, + s.state.Unwrap().softwareName, + s.state.Unwrap().softwareVersion, + req.Input, ) args := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(model.DiscardLogger), From 364bae925800f6ef6508358232f66076a25c7bbc Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 8 Feb 2023 21:33:18 +0000 Subject: [PATCH 14/17] start modifying ooniprobe --- cmd/ooniprobe/internal/nettests/nettests.go | 5 +- .../internal/nettests/nettests_test.go | 2 +- cmd/ooniprobe/internal/nettests/run.go | 21 +- .../internal/nettests/web_connectivity.go | 35 +- cmd/ooniprobe/internal/ooni/ooni.go | 64 ++-- cmd/ooniprobe/internal/ooni/sessionapi.go | 332 ++++++++++++++++++ cmd/ooniprobe/internal/ooni/sessionimpl.go | 174 +++++++++ internal/backendclient/backendclient.go | 10 +- internal/geolocate/cloudflare.go | 1 - internal/geolocate/geolocate.go | 7 + internal/geolocate/iplookup.go | 4 +- internal/geolocate/ubuntu.go | 1 - internal/session/bootstrap.go | 6 +- internal/session/checkin.go | 8 +- internal/session/geolocate.go | 5 + internal/session/state.go | 33 +- internal/session/tunnel.go | 8 +- internal/session/webconnectivity.go | 6 +- internal/sessionresolver/resolver.go | 2 - 19 files changed, 645 insertions(+), 79 deletions(-) create mode 100644 cmd/ooniprobe/internal/ooni/sessionapi.go create mode 100644 cmd/ooniprobe/internal/ooni/sessionimpl.go diff --git a/cmd/ooniprobe/internal/nettests/nettests.go b/cmd/ooniprobe/internal/nettests/nettests.go index bee5be39f7..aaa502ddce 100644 --- a/cmd/ooniprobe/internal/nettests/nettests.go +++ b/cmd/ooniprobe/internal/nettests/nettests.go @@ -10,7 +10,6 @@ import ( "github.com/fatih/color" "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/ooni" "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/output" - engine "github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/model" "github.com/pkg/errors" ) @@ -22,7 +21,7 @@ type Nettest interface { // NewController creates a nettest controller func NewController( - nt Nettest, probe *ooni.Probe, res *model.DatabaseResult, sess *engine.Session) *Controller { + nt Nettest, probe *ooni.Probe, res *model.DatabaseResult, sess ooni.ProbeEngine) *Controller { return &Controller{ Probe: probe, nt: nt, @@ -35,7 +34,7 @@ func NewController( // each nettest instance has one controller type Controller struct { Probe *ooni.Probe - Session *engine.Session + Session ooni.ProbeEngine res *model.DatabaseResult nt Nettest ntCount int diff --git a/cmd/ooniprobe/internal/nettests/nettests_test.go b/cmd/ooniprobe/internal/nettests/nettests_test.go index 0b4d94d703..83ae8e96d3 100644 --- a/cmd/ooniprobe/internal/nettests/nettests_test.go +++ b/cmd/ooniprobe/internal/nettests/nettests_test.go @@ -48,7 +48,7 @@ func TestRun(t *testing.T) { t.Skip("skip test in short mode") } probe := newOONIProbe(t) - sess, err := probe.NewSession(context.Background(), model.RunTypeManual) + sess, err := probe.NewProbeEngine(context.Background(), model.RunTypeManual) if err != nil { t.Fatal(err) } diff --git a/cmd/ooniprobe/internal/nettests/run.go b/cmd/ooniprobe/internal/nettests/run.go index 25385a647e..1c6a8371c3 100644 --- a/cmd/ooniprobe/internal/nettests/run.go +++ b/cmd/ooniprobe/internal/nettests/run.go @@ -9,6 +9,7 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/ooni" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/platform" "github.com/pkg/errors" ) @@ -59,7 +60,7 @@ func RunGroup(config RunGroupConfig) error { return nil } - sess, err := config.Probe.NewSession(context.Background(), config.RunType) + sess, err := config.Probe.NewProbeEngine(context.Background(), config.RunType) if err != nil { log.WithError(err).Error("Failed to create a measurement session") return err @@ -77,7 +78,23 @@ func RunGroup(config RunGroupConfig) error { log.WithError(err).Error("Failed to create the network row") return err } - if err := sess.MaybeLookupBackends(); err != nil { + checkInConfig := &model.OOAPICheckInConfig{ + Charging: true, + OnWiFi: true, + Platform: platform.Name(), + ProbeASN: sess.ProbeASNString(), + ProbeCC: sess.ProbeCC(), + RunType: config.RunType, + SoftwareName: sess.SoftwareName(), + SoftwareVersion: sess.SoftwareVersion(), + WebConnectivity: model.OOAPICheckInConfigWebConnectivity{ + CategoryCodes: config.Probe.Config().Nettests.WebsitesEnabledCategoryCodes, + }, + } + if checkInConfig.WebConnectivity.CategoryCodes == nil { + checkInConfig.WebConnectivity.CategoryCodes = []string{} + } + if err := sess.MaybeLookupBackends(checkInConfig); err != nil { log.WithError(err).Warn("Failed to discover OONI backends") return err } diff --git a/cmd/ooniprobe/internal/nettests/web_connectivity.go b/cmd/ooniprobe/internal/nettests/web_connectivity.go index fc643c0091..6b453f2793 100644 --- a/cmd/ooniprobe/internal/nettests/web_connectivity.go +++ b/cmd/ooniprobe/internal/nettests/web_connectivity.go @@ -4,23 +4,32 @@ import ( "context" "github.com/apex/log" + "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/config" engine "github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/model" ) -func (n WebConnectivity) lookupURLs(ctl *Controller, categories []string) ([]string, error) { - inputloader := &engine.InputLoader{ - CheckInConfig: &model.OOAPICheckInConfig{ - // Setting Charging and OnWiFi to true causes the CheckIn - // API to return to us as much URL as possible with the - // given RunType hint. - Charging: true, - OnWiFi: true, - RunType: ctl.RunType, - WebConnectivity: model.OOAPICheckInConfigWebConnectivity{ - CategoryCodes: categories, - }, +func newCheckInConfig(runType model.RunType, config *config.Config) *model.OOAPICheckInConfig { + result := &model.OOAPICheckInConfig{ + // Setting Charging and OnWiFi to true causes the CheckIn + // API to return to us as much URL as possible with the + // given RunType hint. + Charging: true, + OnWiFi: true, + RunType: runType, + WebConnectivity: model.OOAPICheckInConfigWebConnectivity{ + CategoryCodes: config.Nettests.WebsitesEnabledCategoryCodes, }, + } + if result.WebConnectivity.CategoryCodes == nil { + result.WebConnectivity.CategoryCodes = []string{} + } + return result +} + +func (n WebConnectivity) lookupURLs(ctl *Controller, config *config.Config) ([]string, error) { + inputloader := &engine.InputLoader{ + CheckInConfig: newCheckInConfig(ctl.RunType, config), ExperimentName: "web_connectivity", InputPolicy: model.InputOrQueryBackend, Session: ctl.Session, @@ -40,7 +49,7 @@ type WebConnectivity struct{} // Run starts the test func (n WebConnectivity) Run(ctl *Controller) error { log.Debugf("Enabled category codes are the following %v", ctl.Probe.Config().Nettests.WebsitesEnabledCategoryCodes) - urls, err := n.lookupURLs(ctl, ctl.Probe.Config().Nettests.WebsitesEnabledCategoryCodes) + urls, err := n.lookupURLs(ctl, ctl.Probe.Config()) if err != nil { return err } diff --git a/cmd/ooniprobe/internal/ooni/ooni.go b/cmd/ooniprobe/internal/ooni/ooni.go index dcffdab6c4..c955b68683 100644 --- a/cmd/ooniprobe/internal/ooni/ooni.go +++ b/cmd/ooniprobe/internal/ooni/ooni.go @@ -15,9 +15,9 @@ import ( "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/utils" "github.com/ooni/probe-cli/v3/internal/database" "github.com/ooni/probe-cli/v3/internal/engine" - "github.com/ooni/probe-cli/v3/internal/kvstore" "github.com/ooni/probe-cli/v3/internal/legacy/assetsdir" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/session" "github.com/pkg/errors" ) @@ -41,12 +41,14 @@ type ProbeCLI interface { // ProbeEngine is an instance of the OONI Probe engine. type ProbeEngine interface { + model.LocationProvider + engine.InputLoaderSession Close() error + MaybeLookupBackends(config *model.OOAPICheckInConfig) error MaybeLookupLocation() error - ProbeASNString() string - ProbeCC() string - ProbeIP() string - ProbeNetworkName() string + NewExperimentBuilder(name string) (model.ExperimentBuilder, error) + SoftwareName() string + SoftwareVersion() string } // Probe contains the ooniprobe CLI context. @@ -211,19 +213,11 @@ func (p *Probe) Init(softwareName, softwareVersion, proxy string) error { return nil } -// NewSession creates a new ooni/probe-engine session using the +// newSession creates a new ooni/probe-engine session using the // current configuration inside the context. The caller must close // the session when done using it, by calling sess.Close(). -func (p *Probe) NewSession(ctx context.Context, runType model.RunType) (*engine.Session, error) { - kvstore, err := kvstore.NewFS( - utils.EngineDir(p.home), - ) - if err != nil { - return nil, errors.Wrap(err, "creating engine's kvstore") - } - if err := os.MkdirAll(p.tunnelDir, 0700); err != nil { - return nil, errors.Wrap(err, "creating tunnel dir") - } +func (p *Probe) newSession(ctx context.Context, runType model.RunType) (*engineSession, error) { + // When the software name is the default software name and we're running // in unattended mode, adjust the software name accordingly. // @@ -232,20 +226,38 @@ func (p *Probe) NewSession(ctx context.Context, runType model.RunType) (*engine. if runType == model.RunTypeTimed && softwareName == DefaultSoftwareName { softwareName = DefaultSoftwareName + "-unattended" } - return engine.NewSession(ctx, engine.SessionConfig{ - KVStore: kvstore, - Logger: logger, - SoftwareName: softwareName, - SoftwareVersion: p.softwareVersion, - TempDir: p.tempDir, - TunnelDir: p.tunnelDir, - ProxyURL: p.proxyURL, - }) + + // TODO(bassosimone): unclear to me how to set these fields + // + // - SnowflakeRendezvousMethod + // - TorArgs + // - TorBinary + // - VerboseLogging + + var proxyURL string + if p.proxyURL != nil { + proxyURL = p.proxyURL.String() + } + + config := &session.BootstrapRequest{ + SnowflakeRendezvousMethod: "", + StateDir: utils.EngineDir(p.home), + ProxyURL: proxyURL, + SoftwareName: softwareName, + SoftwareVersion: p.softwareVersion, + TorArgs: nil, + TorBinary: "", + TempDir: p.tempDir, + TunnelDir: p.tunnelDir, + VerboseLogging: false, + } + + return newSession(config, logger), nil } // NewProbeEngine creates a new ProbeEngine instance. func (p *Probe) NewProbeEngine(ctx context.Context, runType model.RunType) (ProbeEngine, error) { - sess, err := p.NewSession(ctx, runType) + sess, err := p.newSession(ctx, runType) if err != nil { return nil, err } diff --git a/cmd/ooniprobe/internal/ooni/sessionapi.go b/cmd/ooniprobe/internal/ooni/sessionapi.go new file mode 100644 index 0000000000..3457b34c49 --- /dev/null +++ b/cmd/ooniprobe/internal/ooni/sessionapi.go @@ -0,0 +1,332 @@ +package ooni + +import ( + "context" + "encoding/json" + "errors" + "os" + "sync" + "time" + + "github.com/ooni/probe-cli/v3/internal/bytecounter" + "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/registry" + "github.com/ooni/probe-cli/v3/internal/session" +) + +// engineSession emulates [engine.engineSession] using [session.engineSession]. +type engineSession struct { + // checkIn stores the most recent check-in API response (if any). + checkIn model.OptionalPtr[model.OOAPICheckInResult] + + // checkInMu protects the checkIn field + checkInMu sync.Mutex + + // config contains the config for bootstrapping the session. + config *session.BootstrapRequest + + // logger is the logger to use. + logger model.Logger + + // once allows us to run cleanups just once. + once sync.Once + + // session is the initially empty session. + session *session.Session +} + +var ( + _ ProbeEngine = &engineSession{} + _ engine.InputLoaderSession = &engineSession{} + _ model.LocationProvider = &engineSession{} +) + +// newSession creates a new instance of Session. +func newSession(config *session.BootstrapRequest, logger model.Logger) *engineSession { + return &engineSession{ + checkIn: model.OptionalPtr[model.OOAPICheckInResult]{}, + checkInMu: sync.Mutex{}, + config: config, + logger: logger, + once: sync.Once{}, + session: session.New(), + } +} + +// SoftwareName implements ProbeEngine +func (s *engineSession) SoftwareName() string { + return s.config.SoftwareName +} + +// SoftwareVersion implements ProbeEngine +func (s *engineSession) SoftwareVersion() string { + return s.config.SoftwareVersion +} + +// Close implements ProbeEngine +func (s *engineSession) Close() error { + s.once.Do(func() { + s.session.Close() + }) + return nil +} + +// MaybeLookupLocation implements ProbeEngine +func (s *engineSession) MaybeLookupLocation() error { + if _, err := s.maybeLookupLocation(); err != nil { + return err + } + return nil +} + +// ProbeASNString implements ProbeEngine +func (s *engineSession) ProbeASNString() string { + location, err := s.maybeLookupLocation() + if err != nil { + return model.DefaultProbeASNString + } + return location.ProbeASNString() +} + +// ProbeCC implements ProbeEngine +func (s *engineSession) ProbeCC() string { + location, err := s.maybeLookupLocation() + if err != nil { + return model.DefaultProbeCC + } + return location.ProbeCC() +} + +// ProbeIP implements ProbeEngine +func (s *engineSession) ProbeIP() string { + location, err := s.maybeLookupLocation() + if err != nil { + return model.DefaultProbeIP + } + return location.ProbeIP() +} + +// ProbeNetworkName implements ProbeEngine +func (s *engineSession) ProbeNetworkName() string { + location, err := s.maybeLookupLocation() + if err != nil { + return model.DefaultProbeNetworkName + } + return location.ProbeNetworkName() +} + +// ProbeASN implements model.LocationProvider +func (s *engineSession) ProbeASN() uint { + location, err := s.maybeLookupLocation() + if err != nil { + return model.DefaultProbeASN + } + return location.ProbeASN() +} + +// ResolverASN implements model.LocationProvider +func (s *engineSession) ResolverASN() uint { + location, err := s.maybeLookupLocation() + if err != nil { + return model.DefaultResolverASN + } + return location.ResolverASN() +} + +// ResolverASNString implements model.LocationProvider +func (s *engineSession) ResolverASNString() string { + location, err := s.maybeLookupLocation() + if err != nil { + return model.DefaultResolverASNString + } + return location.ResolverASNString() +} + +// ResolverIP implements model.LocationProvider +func (s *engineSession) ResolverIP() string { + location, err := s.maybeLookupLocation() + if err != nil { + return model.DefaultResolverIP + } + return location.ResolverIP() +} + +// ResolverNetworkName implements model.LocationProvider +func (s *engineSession) ResolverNetworkName() string { + location, err := s.maybeLookupLocation() + if err != nil { + return model.DefaultResolverNetworkName + } + return location.ResolverNetworkName() +} + +// MaybeLookupBackends implements ProbeEngine +func (s *engineSession) MaybeLookupBackends(config *model.OOAPICheckInConfig) error { + _, err := s.maybeCheckIn(context.Background(), config) + return err +} + +// CheckIn implements engine.InputLoaderSession +func (s *engineSession) CheckIn( + ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResultNettests, error) { + return s.maybeCheckIn(ctx, config) +} + +// NewExperimentBuilder implements ProbeEngine +func (s *engineSession) NewExperimentBuilder(name string) (model.ExperimentBuilder, error) { + factory, err := registry.NewFactory(name) + if err != nil { + return nil, err + } + + // Lock because we are accessing the cached check-in + defer s.checkInMu.Unlock() + s.checkInMu.Lock() + if s.checkIn.IsNone() { + return nil, errors.New("no cached check-in API response") + } + resp := s.checkIn.Unwrap() + + var reportID string + switch name { + case "web_connectivity": + if resp.Tests.WebConnectivity == nil { + return nil, errors.New("no experiment-specific info in check-in API response") + } + reportID = resp.Tests.WebConnectivity.ReportID + default: + return nil, errors.New("not implemented") + } + + meb := &modelExperimentBuilder{ + callbacks: nil, + factory: factory, + reportID: reportID, + session: s, + } + return meb, nil +} + +// modelExperimentBuilder implements [model.ExperimentBuilder] using [session.Session]. +type modelExperimentBuilder struct { + callbacks model.ExperimentCallbacks + factory *registry.Factory + reportID string + session *engineSession +} + +var _ model.ExperimentBuilder = &modelExperimentBuilder{} + +// InputPolicy implements model.ExperimentBuilder +func (meb *modelExperimentBuilder) InputPolicy() model.InputPolicy { + return meb.factory.InputPolicy() +} + +// Interruptible implements model.ExperimentBuilder +func (meb *modelExperimentBuilder) Interruptible() bool { + return meb.factory.Interruptible() +} + +// NewExperiment implements model.ExperimentBuilder +func (meb *modelExperimentBuilder) NewExperiment() model.Experiment { + measurer := meb.factory.NewExperimentMeasurer() + me := &modelExperiment{ + bc: bytecounter.New(), + measurer: measurer, + meb: meb, + testStartTime: time.Now(), + } + return me +} + +// Options implements model.ExperimentBuilder +func (meb *modelExperimentBuilder) Options() (map[string]model.ExperimentOptionInfo, error) { + return meb.factory.Options() +} + +// SetCallbacks implements model.ExperimentBuilder +func (meb *modelExperimentBuilder) SetCallbacks(callbacks model.ExperimentCallbacks) { + meb.callbacks = callbacks +} + +// SetOptionAny implements model.ExperimentBuilder +func (meb *modelExperimentBuilder) SetOptionAny(key string, value any) error { + return meb.factory.SetOptionAny(key, value) +} + +// SetOptionsAny implements model.ExperimentBuilder +func (meb *modelExperimentBuilder) SetOptionsAny(options map[string]any) error { + return meb.factory.SetOptionsAny(options) +} + +// modelExperiment implements [model.Experiment] using [session.Session]. +type modelExperiment struct { + bc *bytecounter.Counter + measurer model.ExperimentMeasurer + meb *modelExperimentBuilder + testStartTime time.Time +} + +var _ model.Experiment = &modelExperiment{} + +// GetSummaryKeys implements model.Experiment +func (me *modelExperiment) GetSummaryKeys(m *model.Measurement) (any, error) { + return me.measurer.GetSummaryKeys(m) +} + +// KibiBytesReceived implements model.Experiment +func (me *modelExperiment) KibiBytesReceived() float64 { + return me.bc.KibiBytesReceived() +} + +// KibiBytesSent implements model.Experiment +func (me *modelExperiment) KibiBytesSent() float64 { + return me.bc.KibiBytesSent() +} + +// MeasureAsync implements model.Experiment +func (me *modelExperiment) MeasureAsync(ctx context.Context, input string) (<-chan *model.Measurement, error) { + return nil, errors.New("not implemented") +} + +// MeasureWithContext implements model.Experiment +func (me *modelExperiment) MeasureWithContext(ctx context.Context, input string) (*model.Measurement, error) { + // XXX: bytes sent and received? + switch me.measurer.ExperimentName() { + case "web_connectivity": + return me.runWebConnectivity(ctx, input) + default: + return nil, errors.New("not implemented") + } +} + +// Name implements model.Experiment +func (me *modelExperiment) Name() string { + return me.measurer.ExperimentName() +} + +// OpenReportContext implements model.Experiment +func (me *modelExperiment) OpenReportContext(ctx context.Context) error { + return nil +} + +// ReportID implements model.Experiment +func (me *modelExperiment) ReportID() string { + return me.meb.reportID +} + +// SaveMeasurement implements model.Experiment +func (me *modelExperiment) SaveMeasurement(measurement *model.Measurement, filePath string) error { + data, err := json.Marshal(measurement) + if err != nil { + return err + } + return os.WriteFile(filePath, data, 0600) +} + +// SubmitAndUpdateMeasurementContext implements model.Experiment +func (me *modelExperiment) SubmitAndUpdateMeasurementContext(ctx context.Context, measurement *model.Measurement) error { + // Note: the measurement has already the correct reportID since the beginning + return me.submit(ctx, measurement) +} diff --git a/cmd/ooniprobe/internal/ooni/sessionimpl.go b/cmd/ooniprobe/internal/ooni/sessionimpl.go new file mode 100644 index 0000000000..a3d6825a85 --- /dev/null +++ b/cmd/ooniprobe/internal/ooni/sessionimpl.go @@ -0,0 +1,174 @@ +package ooni + +import ( + "context" + + "github.com/ooni/probe-cli/v3/internal/geolocate" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/session" +) + +// maybeBootstrap performs the session's bootstrap. +func (s *engineSession) maybeBootstrap() error { + ctx := context.Background() // XXX + if err := s.session.Send(ctx, &session.Request{Bootstrap: s.config}); err != nil { + return err + } + for { + resp, err := s.session.Recv(ctx) + if err != nil { + return err + } + if resp.Log != nil { + s.emitLog(resp.Log) + continue + } + if resp.Ticker != nil { + s.logger.Infof("bootstrap in progress (elapsed: %+v)", resp.Ticker.ElapsedTime) + continue + } + if resp.Bootstrap != nil { + return resp.Bootstrap.Error + } + s.logger.Warnf("unexpected event: %+v", resp) + } +} + +// maybeLookupLocation geolocates the probe. +func (s *engineSession) maybeLookupLocation() (*geolocate.Results, error) { + ctx := context.Background() // XXX + if err := s.maybeBootstrap(); err != nil { + return nil, err + } + req := &session.GeolocateRequest{} + if err := s.session.Send(ctx, &session.Request{Geolocate: req}); err != nil { + return nil, err + } + for { + resp, err := s.session.Recv(ctx) + if err != nil { + return nil, err + } + if resp.Log != nil { + s.emitLog(resp.Log) + continue + } + if resp.Ticker != nil { + s.logger.Infof("geolocate in progress (elapsed: %+v)", resp.Ticker.ElapsedTime) + continue + } + if resp.Geolocate != nil { + return resp.Geolocate.Location, resp.Geolocate.Error + } + s.logger.Warnf("unexpected event: %+v", resp) + } +} + +// maybeCheckIn calls the check-in API. +func (s *engineSession) maybeCheckIn( + ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResultNettests, error) { + if err := s.maybeBootstrap(); err != nil { + return nil, err + } + if err := s.session.Send(ctx, &session.Request{CheckIn: config}); err != nil { + return nil, err + } + for { + resp, err := s.session.Recv(ctx) + if err != nil { + return nil, err + } + if resp.Log != nil { + s.emitLog(resp.Log) + continue + } + if resp.Ticker != nil { + s.logger.Infof("check-in in progress (elapsed: %+v)", resp.Ticker.ElapsedTime) + continue + } + if resp.CheckIn != nil { + if resp.CheckIn.Error != nil { + return nil, resp.CheckIn.Error + } + // While this code has been writting with single-goroutine usage + // in mind, it seems safer to protect this variable anyway + s.checkInMu.Lock() + s.checkIn = model.NewOptionalPtr(resp.CheckIn.Result) + s.checkInMu.Unlock() + return &resp.CheckIn.Result.Tests, nil + } + s.logger.Warnf("unexpected event: %+v", resp) + } +} + +// emitLog emits a log event. +func (s *engineSession) emitLog(ev *session.LogEvent) { + switch ev.Level { + case "DEBUG": + s.logger.Debug(ev.Message) + case "WARNING": + s.logger.Warn(ev.Message) + default: + s.logger.Info(ev.Message) + } +} + +// runWebConnectivity runs the Web Connectivity experiment. +func (me *modelExperiment) runWebConnectivity( + ctx context.Context, input string) (*model.Measurement, error) { + runtimex.Assert(me.measurer.ExperimentName() == "web_connectivity", "invalid experiment") + req := &session.WebConnectivityRequest{ + Input: input, + ReportID: me.meb.reportID, + TestStartTime: me.testStartTime, + } + sess := me.meb.session.session + if err := sess.Send(ctx, &session.Request{WebConnectivity: req}); err != nil { + return nil, err + } + for { + resp, err := sess.Recv(ctx) + if err != nil { + return nil, err + } + if resp.Log != nil { + me.meb.session.emitLog(resp.Log) + continue + } + if resp.Ticker != nil { + me.meb.session.logger.Infof("check-in in progress (elapsed: %+v)", resp.Ticker.ElapsedTime) + continue + } + if resp.WebConnectivity != nil { + return resp.WebConnectivity.Measurement, resp.WebConnectivity.Error + } + me.meb.session.logger.Warnf("unexpected event: %+v", resp) + } +} + +// submit submites a measurement. +func (me *modelExperiment) submit(ctx context.Context, measurement *model.Measurement) error { + sess := me.meb.session.session + if err := sess.Send(ctx, &session.Request{Submit: measurement}); err != nil { + return err + } + for { + resp, err := sess.Recv(ctx) + if err != nil { + return err + } + if resp.Log != nil { + me.meb.session.emitLog(resp.Log) + continue + } + if resp.Ticker != nil { + me.meb.session.logger.Infof("webconnectivity (elapsed: %+v)", resp.Ticker.ElapsedTime) + continue + } + if resp.Submit != nil { + return resp.Submit.Error + } + me.meb.session.logger.Warnf("unexpected event: %+v", resp) + } +} diff --git a/internal/backendclient/backendclient.go b/internal/backendclient/backendclient.go index a6bbe87227..7b46fc85f7 100644 --- a/internal/backendclient/backendclient.go +++ b/internal/backendclient/backendclient.go @@ -8,6 +8,7 @@ import ( "net/url" "github.com/ooni/probe-cli/v3/internal/httpapi" + "github.com/ooni/probe-cli/v3/internal/measurexlite" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/ooapi" ) @@ -58,7 +59,14 @@ func New(config *Config) *Client { // CheckIn invokes the check-in API. func (c *Client) CheckIn( ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResult, error) { - return httpapi.Call(ctx, ooapi.NewDescriptorCheckIn(config), c.endpoint) + op := measurexlite.NewOperationLogger( + c.endpoint.Logger, + "backendclient: check-in using %s", + c.endpoint.BaseURL, + ) + r, err := httpapi.Call(ctx, ooapi.NewDescriptorCheckIn(config), c.endpoint) + op.Stop(err) + return r, err } // FetchPsiphonConfig retrieves Psiphon configuration. diff --git a/internal/geolocate/cloudflare.go b/internal/geolocate/cloudflare.go index 3de61a538b..4cbc611f42 100644 --- a/internal/geolocate/cloudflare.go +++ b/internal/geolocate/cloudflare.go @@ -28,6 +28,5 @@ func cloudflareIPLookup( } r := regexp.MustCompile("(?:ip)=(.*)") ip := strings.Trim(string(r.Find(data)), "ip=") - logger.Debugf("cloudflare: body: %s", ip) return ip, nil } diff --git a/internal/geolocate/geolocate.go b/internal/geolocate/geolocate.go index 6805a9ff7d..9cf8c300f0 100644 --- a/internal/geolocate/geolocate.go +++ b/internal/geolocate/geolocate.go @@ -129,6 +129,7 @@ func NewTask(config Config) *Task { } return &Task{ countryLookupper: mmdbLookupper{}, + logger: config.Logger, probeIPLookupper: ipLookupClient(config), probeASNLookupper: mmdbLookupper{}, resolverASNLookupper: mmdbLookupper{}, @@ -142,6 +143,7 @@ func NewTask(config Config) *Task { // instance of Task using the NewTask factory. type Task struct { countryLookupper countryLookupper + logger model.Logger probeIPLookupper probeIPLookupper probeASNLookupper asnLookupper resolverASNLookupper asnLookupper @@ -169,12 +171,15 @@ func (op Task) Run(ctx context.Context) (*Results, error) { if err != nil { return out, fmt.Errorf("lookupASN failed: %w", err) } + op.logger.Infof("geolocate: probe ASN: %d", asn) + op.logger.Infof("geolocate: probe network name: %s", networkName) out.ASN = asn out.NetworkName = networkName cc, err := op.countryLookupper.LookupCC(out.IPAddr) if err != nil { return out, fmt.Errorf("lookupProbeCC failed: %w", err) } + op.logger.Infof("geolocate: country code: %s", cc) out.CountryCode = cc out.didResolverLookup = true // Note: ignoring the result of lookupResolverIP and lookupASN @@ -192,6 +197,8 @@ func (op Task) Run(ctx context.Context) (*Results, error) { if err != nil { return out, nil // intentional } + op.logger.Infof("geolocate: resolver ASN: %d", resolverASN) + op.logger.Infof("geolocate: resolver network name: %s", resolverNetworkName) out.ResolverASNumber = resolverASN out.ResolverASNetworkName = resolverNetworkName return out, nil diff --git a/internal/geolocate/iplookup.go b/internal/geolocate/iplookup.go index 3d9711f323..47b23a55c4 100644 --- a/internal/geolocate/iplookup.go +++ b/internal/geolocate/iplookup.go @@ -97,14 +97,14 @@ func (c ipLookupClient) doWithCustomFunc( if net.ParseIP(ip) == nil { return model.DefaultProbeIP, fmt.Errorf("%w: %s", ErrInvalidIPAddress, ip) } - c.Logger.Debugf("iplookup: IP: %s", ip) + c.Logger.Debugf("geolocate: IP: %s", ip) return ip, nil } func (c ipLookupClient) LookupProbeIP(ctx context.Context) (string, error) { union := multierror.New(ErrAllIPLookuppersFailed) for _, method := range makeSlice() { - c.Logger.Infof("iplookup: using %s", method.name) + c.Logger.Infof("geolocate: using %s", method.name) ip, err := c.doWithCustomFunc(ctx, method.fn) if err == nil { return ip, nil diff --git a/internal/geolocate/ubuntu.go b/internal/geolocate/ubuntu.go index 0f75afb4e0..2556470d4f 100644 --- a/internal/geolocate/ubuntu.go +++ b/internal/geolocate/ubuntu.go @@ -30,7 +30,6 @@ func ubuntuIPLookup( if err != nil { return model.DefaultProbeIP, err } - logger.Debugf("ubuntu: body: %s", string(data)) var v ubuntuResponse err = xml.Unmarshal(data, &v) if err != nil { diff --git a/internal/session/bootstrap.go b/internal/session/bootstrap.go index f7b2243d48..7581bba084 100644 --- a/internal/session/bootstrap.go +++ b/internal/session/bootstrap.go @@ -6,7 +6,6 @@ package session import ( "context" - "errors" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/runtimex" @@ -85,13 +84,10 @@ func (s *Session) bootstrap(ctx context.Context, req *BootstrapRequest) { }) } -// ErrAlreadyBootstrapped indicates that we already bootstrapped a [Session]. -var ErrAlreadyBootstrapped = errors.New("session: already bootstrapped") - // dobootstrap implements bootstrap. func (s *Session) dobootstrap(ctx context.Context, req *BootstrapRequest) error { if s.state.IsSome() { - return ErrAlreadyBootstrapped + return nil // idempotent } state, err := s.newState(ctx, req) if err != nil { diff --git a/internal/session/checkin.go b/internal/session/checkin.go index b2bb02e882..791b892688 100644 --- a/internal/session/checkin.go +++ b/internal/session/checkin.go @@ -40,11 +40,17 @@ func (s *Session) docheckin(ctx context.Context, req *CheckInRequest) (*model.OO if s.state.IsNone() { return nil, ErrNotBootstrapped } + if s.state.Unwrap().checkIn.IsSome() { + // TODO(bassosimone): in the future we should define caching + // policies for the check-in response, but for now this is fine. + return s.state.Unwrap().checkIn.Unwrap(), nil + } ts := newTickerService(ctx, s) defer ts.stop() - result, err := s.state.Unwrap().backendClient.CheckIn(ctx, req) + backendClient := s.state.Unwrap().backendClient + result, err := backendClient.CheckIn(ctx, req) if err != nil { return nil, err } diff --git a/internal/session/geolocate.go b/internal/session/geolocate.go index 49a7028d21..9a0d70a710 100644 --- a/internal/session/geolocate.go +++ b/internal/session/geolocate.go @@ -45,6 +45,11 @@ func (s *Session) dogeolocate(ctx context.Context, req *GeolocateRequest) (*geol if s.state.IsNone() { return nil, ErrNotBootstrapped } + if s.state.Unwrap().location.IsSome() { + // TODO(bassosimone): in the future we should define caching + // policies for the location, but for now this is fine. + return s.state.Unwrap().location.Unwrap(), nil + } ts := newTickerService(ctx, s) defer ts.stop() diff --git a/internal/session/state.go b/internal/session/state.go index a8bac2e63e..b1353c795c 100644 --- a/internal/session/state.go +++ b/internal/session/state.go @@ -22,7 +22,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/version" ) -// state is the boostrapped [Session] state. Only the background +// state is the bootstrapped [Session] state. Only the background // goroutine is allowed to manipulate the [state]. type state struct { // backendClient is the backend client we're using. @@ -108,10 +108,16 @@ func (s *Session) newState(ctx context.Context, req *BootstrapRequest) (*state, ts := newTickerService(ctx, s) defer ts.stop() - logger.Infof("creating key-value store at %s", req.StateDir) + logger.Infof("bootstrap: creating key-value store at %s", req.StateDir) kvstore, err := kvstore.NewFS(req.StateDir) if err != nil { - logger.Warnf("cannot create key-value store: %s", err.Error()) + logger.Warnf("bootstrap: cannot create key-value store: %s", err.Error()) + return nil, err + } + + logger.Infof("bootstrap: creating tunnels dir at %s", req.TunnelDir) + if err := os.MkdirAll(req.TunnelDir, 0700); err != nil { + logger.Warnf("bootstrap: cannot create tunnels dir: %s", err.Error()) return nil, err } @@ -121,10 +127,9 @@ func (s *Session) newState(ctx context.Context, req *BootstrapRequest) (*state, return nil, err } - logger.Infof("checking whether we need to create a circumvention tunnel") tunnel, err := newTunnel(ctx, logger, req) if err != nil { - logger.Warnf("cannot create tunnel: %s", err.Error()) + logger.Warnf("bootstrap: cannot create tunnel: %s", err.Error()) return nil, err } @@ -153,10 +158,10 @@ func newStateCannotFail( runtimex.Assert(tempDir != "", "passed an empty tempDir") runtimex.Assert(req != nil, "passed a nil req") - logger.Infof("creating a session byte counter") + logger.Infof("bootstrap: creating a session byte counter") counter := bytecounter.New() - logger.Infof("creating a resolver for the session") + logger.Infof("bootstrap: creating a resolver for the session") resolver := &sessionresolver.Resolver{ ByteCounter: counter, KVStore: kvstore, @@ -164,7 +169,7 @@ func newStateCannotFail( ProxyURL: tunnel.SOCKS5ProxyURL(), // possibly nil, which is OK } - logger.Infof("creating an HTTP client for the session") + logger.Infof("bootstrap: creating an HTTP client for the session") httpClient := sessionhttpclient.New(&sessionhttpclient.Config{ ByteCounter: counter, Logger: logger, @@ -172,7 +177,7 @@ func newStateCannotFail( Resolver: resolver, }) - logger.Infof("creating the default user-agent string") + logger.Infof("bootstrap: creating the default user-agent string") userAgent := fmt.Sprintf( "%s/%s ooniprobe-engine/%s", req.SoftwareName, @@ -180,7 +185,7 @@ func newStateCannotFail( version.Version, ) - logger.Infof("creating a client to communicate with the OONI backend") + logger.Infof("bootstrap: creating an OONI backend client") backendClient := backendclient.New(&backendclient.Config{ BaseURL: nil, // use the default KVStore: kvstore, @@ -189,7 +194,7 @@ func newStateCannotFail( UserAgent: userAgent, }) - logger.Infof("session bootstrap complete") + logger.Infof("bootstrap: complete") state := &state{ backendClient: backendClient, checkIn: model.OptionalPtr[model.OOAPICheckInResult]{}, @@ -212,14 +217,14 @@ func newStateCannotFail( // stateNewTempDir creates a new temporary directory for [state]. func stateNewTempDir(logger model.Logger, req *BootstrapRequest) (string, error) { - logger.Infof("creating temporary directory inside %s", req.TempDir) + logger.Infof("bootstrap: creating temporary directory inside %s", req.TempDir) if err := os.MkdirAll(req.TempDir, 0700); err != nil { - logger.Warnf("cannot create temporary directory root: %s", err.Error()) + logger.Warnf("bootstrap: cannot create temporary directory root: %s", err.Error()) return "", err } tempDir, err := os.MkdirTemp(req.TempDir, "") if err != nil { - logger.Warnf("cannot create session temporary directory: %s", err.Error()) + logger.Warnf("bootstrap: cannot create session temporary directory: %s", err.Error()) return "", err } return tempDir, nil diff --git a/internal/session/tunnel.go b/internal/session/tunnel.go index 953649dfb8..c27e582fa2 100644 --- a/internal/session/tunnel.go +++ b/internal/session/tunnel.go @@ -22,7 +22,7 @@ var ErrUnsupportedTunnelScheme = errors.New("session: unsupported tunnel scheme" // newTunnel creates a new [tunnel.Tunnel] given a ProxyURL. func newTunnel(ctx context.Context, logger model.Logger, req *BootstrapRequest) (tunnel.Tunnel, error) { if req.ProxyURL == "" { - logger.Info("no need to create any tunnel") + logger.Info("bootstrap: no need to create any tunnel") return &nullTunnel{}, nil } URL, err := url.Parse(req.ProxyURL) @@ -31,13 +31,13 @@ func newTunnel(ctx context.Context, logger model.Logger, req *BootstrapRequest) } switch scheme := URL.Scheme; scheme { case "socks5": - logger.Infof("creating fake tunnel for %s", req.ProxyURL) + logger.Infof("bootstrap: creating fake tunnel for %s", req.ProxyURL) return &socks5Tunnel{URL}, nil case "tor", "torsf": - logger.Infof("creating %s tunnel; please, be patient...", scheme) + logger.Infof("bootstrap: creating %s tunnel; please, be patient...", scheme) return newTorOrTorsfTunnel(ctx, logger, req, scheme) case "psiphon": - logger.Info("creating psiphon tunnel; please, be patient...") + logger.Info("bootstrap: creating psiphon tunnel; please, be patient...") return newPsiphonTunnel(ctx, logger, req) default: return nil, fmt.Errorf("%w: %s", ErrUnsupportedTunnelScheme, scheme) diff --git a/internal/session/webconnectivity.go b/internal/session/webconnectivity.go index 9760eb745a..06d9297336 100644 --- a/internal/session/webconnectivity.go +++ b/internal/session/webconnectivity.go @@ -8,7 +8,7 @@ import ( "context" "time" - "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivitylte" + "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivity" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/runtimex" ) @@ -64,8 +64,8 @@ func (s *Session) dowebconnectivity( return nil, err } - cfg := &webconnectivitylte.Config{} - runner := webconnectivitylte.NewExperimentMeasurer(cfg) + cfg := webconnectivity.Config{} + runner := webconnectivity.NewExperimentMeasurer(cfg) measurement := model.NewMeasurement( adapter.location, runner.ExperimentName(), diff --git a/internal/sessionresolver/resolver.go b/internal/sessionresolver/resolver.go index 6181a05dc3..0b0b124ae9 100644 --- a/internal/sessionresolver/resolver.go +++ b/internal/sessionresolver/resolver.go @@ -13,7 +13,6 @@ import ( "sync" "time" - "github.com/apex/log" "github.com/ooni/probe-cli/v3/internal/bytecounter" "github.com/ooni/probe-cli/v3/internal/measurexlite" "github.com/ooni/probe-cli/v3/internal/model" @@ -101,7 +100,6 @@ var ErrLookupHost = errors.New("sessionresolver: LookupHost failed") // multierror.Union error on failure, so you can see individual errors // and get a better picture of what's been going wrong. func (r *Resolver) LookupHost(ctx context.Context, hostname string) ([]string, error) { - log.Warnf("PROXY URL: %+v", r.ProxyURL) state := r.readstatedefault() r.maybeConfusion(state, time.Now().UnixNano()) defer r.writestate(state) From 8402b7b18c6b9d997684b59daa187501376e8b24 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 8 Feb 2023 23:00:39 +0000 Subject: [PATCH 15/17] poc: new code for running experiments for ooniprobe --- .../webconnectivity/webconnectivity.go | 16 +- internal/nettests/doc.go | 2 + internal/nettests/experiment.go | 44 ++++++ internal/nettests/inputloader.go | 77 +++++++++ internal/nettests/session.go | 143 +++++++++++++++++ internal/nettests/utils.go | 17 ++ internal/nettests/webconnectivity.go | 146 ++++++++++++++++++ 7 files changed, 441 insertions(+), 4 deletions(-) create mode 100644 internal/nettests/doc.go create mode 100644 internal/nettests/experiment.go create mode 100644 internal/nettests/inputloader.go create mode 100644 internal/nettests/session.go create mode 100644 internal/nettests/utils.go create mode 100644 internal/nettests/webconnectivity.go diff --git a/internal/experiment/webconnectivity/webconnectivity.go b/internal/experiment/webconnectivity/webconnectivity.go index 40732abcb9..abd8c76b2a 100644 --- a/internal/experiment/webconnectivity/webconnectivity.go +++ b/internal/experiment/webconnectivity/webconnectivity.go @@ -14,8 +14,11 @@ import ( ) const ( - testName = "web_connectivity" - testVersion = "0.4.2" + // ExperimentName is the experiment name + ExperimentName = "web_connectivity" + + // ExperimentVersion is the experiment version + ExperimentVersion = "0.4.2" ) // Config contains the experiment config. @@ -85,12 +88,12 @@ func NewExperimentMeasurer(config Config) model.ExperimentMeasurer { // ExperimentName implements ExperimentMeasurer.ExperExperimentName. func (m Measurer) ExperimentName() string { - return testName + return ExperimentName } // ExperimentVersion implements ExperimentMeasurer.ExperExperimentVersion. func (m Measurer) ExperimentVersion() string { - return testVersion + return ExperimentVersion } var ( @@ -261,6 +264,11 @@ type SummaryKeys struct { // GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) { + return GetSummaryKeys(measurement) +} + +// GetSummaryKeys returns the Web Connectivity summary keys +func GetSummaryKeys(measurement *model.Measurement) (interface{}, error) { sk := SummaryKeys{IsAnomaly: false} tk, ok := measurement.TestKeys.(*TestKeys) if !ok { diff --git a/internal/nettests/doc.go b/internal/nettests/doc.go new file mode 100644 index 0000000000..4c4482c063 --- /dev/null +++ b/internal/nettests/doc.go @@ -0,0 +1,2 @@ +// Package nettests contains the nettests implementation. +package nettests diff --git a/internal/nettests/experiment.go b/internal/nettests/experiment.go new file mode 100644 index 0000000000..7db1011144 --- /dev/null +++ b/internal/nettests/experiment.go @@ -0,0 +1,44 @@ +package nettests + +import ( + "errors" + + "github.com/ooni/probe-cli/v3/internal/model" + "golang.org/x/net/context" +) + +// ExperimentFactory constructs a generic [Experiment]. +type ExperimentFactory interface { + // NewExperiment creates a new instance of [Experiment]. + NewExperiment(callbacks model.ExperimentCallbacks) Experiment +} + +// Experiment is the generic API of all experiments. +type Experiment interface { + // GetSummaryKeys returns a data structure containing a + // summary of the test keys for ooniprobe. + GetSummaryKeys(m *model.Measurement) (any, error) + + // KibiBytesReceived accounts for the KibiBytes received by the experiment. + KibiBytesReceived() float64 + + // KibiBytesSent is like KibiBytesReceived but for the bytes sent. + KibiBytesSent() float64 + + // Measure performs a synchronous measurement and returns + // either a valid measurement or an error. + Measure(ctx context.Context, input string) (*model.Measurement, error) + + // Name returns the experiment name. + Name() string + + // ReportID returns the reportID used by this experiment. + ReportID() string + + // Session returns the session owning this experiment. + Session() *Session +} + +// ErrMissingCheckInConfig is the error returned when we do not have +// a check-in configuration for the selected experiment. +var ErrMissingCheckInConfig = errors.New("nettests: missing check-in config") diff --git a/internal/nettests/inputloader.go b/internal/nettests/inputloader.go new file mode 100644 index 0000000000..d548e6bb50 --- /dev/null +++ b/internal/nettests/inputloader.go @@ -0,0 +1,77 @@ +package nettests + +import ( + "bufio" + "errors" + "fmt" + + "github.com/ooni/probe-cli/v3/internal/fsx" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// ErrDetectedEmptyFile indicates we detected an open file. +var ErrDetectedEmptyFile = errors.New("inputloader: file did not contain any input") + +// ErrNoAvailableInput indicates there's no available input. +var ErrNoAvailableInput = errors.New("inputloader: no available input") + +// loadInputs loads inputs by giving priority to user-defined +// input and falling back to API-provided inputs. +func loadInputs( + apiInputs []model.OOAPIURLInfo, + userFiles []string, + userInputs []string, +) ([]model.OOAPIURLInfo, error) { + inputs, err := loadLocalInputs(userFiles, userInputs) + if err != nil || len(inputs) > 0 { + return inputs, err + } + if len(apiInputs) <= 0 { + return nil, ErrNoAvailableInput + } + return apiInputs, nil +} + +// loadLocalInputs loads inputs from user-provided inputs and files. +func loadLocalInputs(userFiles, userInputs []string) ([]model.OOAPIURLInfo, error) { + inputs := []model.OOAPIURLInfo{} + for _, input := range userInputs { + inputs = append(inputs, model.OOAPIURLInfo{URL: input}) + } + for _, filepath := range userFiles { + extra, err := loadLocalInputFile(filepath) + if err != nil { + return nil, err + } + // See https://github.com/ooni/probe-engine/issues/1123. + if len(extra) <= 0 { + return nil, fmt.Errorf("%w: %s", ErrDetectedEmptyFile, filepath) + } + inputs = append(inputs, extra...) + } + return inputs, nil +} + +// loadLocalInputFile reads inputs from the specified file. +func loadLocalInputFile(filepath string) ([]model.OOAPIURLInfo, error) { + inputs := []model.OOAPIURLInfo{} + filep, err := fsx.OpenFile(filepath) + if err != nil { + return nil, err + } + defer filep.Close() + // Implementation note: when you save file with vim, you have newline at + // end of file and you don't want to consider that an input line. While there + // ignore any other empty line that may occur inside the file. + scanner := bufio.NewScanner(filep) + for scanner.Scan() { + line := scanner.Text() + if line != "" { + inputs = append(inputs, model.OOAPIURLInfo{URL: line}) + } + } + if scanner.Err() != nil { + return nil, scanner.Err() + } + return inputs, nil +} diff --git a/internal/nettests/session.go b/internal/nettests/session.go new file mode 100644 index 0000000000..812194caa3 --- /dev/null +++ b/internal/nettests/session.go @@ -0,0 +1,143 @@ +package nettests + +import ( + "sync" + + "github.com/ooni/probe-cli/v3/internal/geolocate" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/session" + "golang.org/x/net/context" +) + +// Session is a measurement session. +type Session struct { + // bootstrapRequest contains settings to bootstrap a session. + bootstrapRequest *session.BootstrapRequest + + // logger is the logger to use. + logger model.Logger + + // once allows us to run cleanups just once. + once sync.Once + + // session is the initially empty session. + session *session.Session +} + +// NewSession creates a new [Session] instance. +func NewSession(request *session.BootstrapRequest, logger model.Logger) *Session { + return &Session{ + bootstrapRequest: request, + logger: logger, + once: sync.Once{}, + session: session.New(), + } +} + +// Close implements ProbeEngine +func (s *Session) Close() error { + s.once.Do(func() { + s.session.Close() + }) + return nil +} + +// Bootstrap performs the session bootstrap. +func (s *Session) Bootstrap(ctx context.Context) error { + if err := s.session.Send(ctx, &session.Request{Bootstrap: s.bootstrapRequest}); err != nil { + return err + } + for { + resp, err := s.session.Recv(ctx) + if err != nil { + return err + } + if resp.Log != nil { + s.emitLog(resp.Log) + continue + } + if resp.Bootstrap != nil { + return resp.Bootstrap.Error + } + } +} + +// Geolocate runs the geolocate task. +func (s *Session) Geolocate(ctx context.Context) (*geolocate.Results, error) { + if err := s.Bootstrap(ctx); err != nil { + return nil, err + } + req := &session.GeolocateRequest{} + if err := s.session.Send(ctx, &session.Request{Geolocate: req}); err != nil { + return nil, err + } + for { + resp, err := s.session.Recv(ctx) + if err != nil { + return nil, err + } + if resp.Log != nil { + s.emitLog(resp.Log) + continue + } + if resp.Geolocate != nil { + return resp.Geolocate.Location, resp.Geolocate.Error + } + } +} + +// CheckIn runs the checkIn task. +func (s *Session) CheckIn( + ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResult, error) { + if err := s.Bootstrap(ctx); err != nil { + return nil, err + } + if err := s.session.Send(ctx, &session.Request{CheckIn: config}); err != nil { + return nil, err + } + for { + resp, err := s.session.Recv(ctx) + if err != nil { + return nil, err + } + if resp.Log != nil { + s.emitLog(resp.Log) + continue + } + if resp.CheckIn != nil { + return resp.CheckIn.Result, resp.CheckIn.Error + } + } +} + +// Submit submits the given measurement. +func (s *Session) Submit(ctx context.Context, measurement *model.Measurement) error { + if err := s.session.Send(ctx, &session.Request{Submit: measurement}); err != nil { + return err + } + for { + resp, err := s.session.Recv(ctx) + if err != nil { + return err + } + if resp.Log != nil { + s.emitLog(resp.Log) + continue + } + if resp.Submit != nil { + return resp.Submit.Error + } + } +} + +// emitLog emits a log event. +func (s *Session) emitLog(ev *session.LogEvent) { + switch ev.Level { + case "DEBUG": + s.logger.Debug(ev.Message) + case "WARNING": + s.logger.Warn(ev.Message) + default: + s.logger.Info(ev.Message) + } +} diff --git a/internal/nettests/utils.go b/internal/nettests/utils.go new file mode 100644 index 0000000000..e0824859d4 --- /dev/null +++ b/internal/nettests/utils.go @@ -0,0 +1,17 @@ +package nettests + +import ( + "encoding/json" + "os" + + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// SaveMeasurement saves the given measurement in the given directory. +func SaveMeasurement(measurement *model.Measurement, filename string) error { + data, err := json.Marshal(measurement) + runtimex.PanicOnError(err, "json.Marshal failed") + data = append(data, byte('\n')) + return os.WriteFile(filename, data, 0600) +} diff --git a/internal/nettests/webconnectivity.go b/internal/nettests/webconnectivity.go new file mode 100644 index 0000000000..b830895ac9 --- /dev/null +++ b/internal/nettests/webconnectivity.go @@ -0,0 +1,146 @@ +package nettests + +import ( + "context" + "time" + + "github.com/ooni/probe-cli/v3/internal/bytecounter" + "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivity" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/session" +) + +// WebConnectivityFactoryConfig contains config for creating +// a new [WebConnectivityFactory] instance. +type WebConnectivityFactoryConfig struct { + // CheckIn contains the MANDATORY response returned by check-in. + CheckIn *model.OOAPICheckInResult + + // InputFiles contains the OPTIONAL input files to read. + InputFiles []string + + // Inputs contains the OPTIONAL inputs to read. + Inputs []string + + // Session is the MANDATORY session. + Session *Session +} + +// WebConnectivityFactory creates the [WebConnectivity] experiment. The zero +// value is invalid. Construct using [NewWebConnectivityFactory]. +type WebConnectivityFactory struct { + experiment *WebConnectivity +} + +// NewWebConnectivityFactory creates a new [WebConnectivity] factory. +func NewWebConnectivityFactory(config *WebConnectivityFactoryConfig) (*WebConnectivityFactory, error) { + runtimex.Assert(config != nil, "passed nil config") + runtimex.Assert(config.CheckIn != nil, "passed nil config.CheckIn") + runtimex.Assert(config.Session != nil, "passed nil config.Session") + if config.CheckIn.Tests.WebConnectivity == nil { + return nil, ErrMissingCheckInConfig + } + f := &WebConnectivityFactory{ + experiment: &WebConnectivity{ + byteCounter: bytecounter.New(), + config: config, + callbacks: model.NewPrinterCallbacks(model.DiscardLogger), + testStartTime: time.Now(), + }, + } + return f, nil +} + +var _ ExperimentFactory = &WebConnectivityFactory{} + +// LoadInputs loads the proper inputs for this experiment. +func (f *WebConnectivityFactory) LoadInputs() ([]model.OOAPIURLInfo, error) { + return loadInputs( + f.experiment.config.CheckIn.Tests.WebConnectivity.URLs, + f.experiment.config.InputFiles, + f.experiment.config.Inputs, + ) +} + +// NewExperiment implements ExperimentFactory +func (f *WebConnectivityFactory) NewExperiment(callbacks model.ExperimentCallbacks) Experiment { + f.experiment.callbacks = callbacks + return f.experiment +} + +// WebConnectivity is the Web Connectivity experiment. The zero value +// is invalid. Construct using [WebConnectivityFactory]. +type WebConnectivity struct { + // byteCounter is the byte counter we use. + byteCounter *bytecounter.Counter + + // config is the config with which we were created. + config *WebConnectivityFactoryConfig + + // callbacks contains the experiment callbacks. + callbacks model.ExperimentCallbacks + + // testStartTime is when we started this test. + testStartTime time.Time +} + +var _ Experiment = &WebConnectivity{} + +// GetSummaryKeys implements Experiment +func (e *WebConnectivity) GetSummaryKeys(m *model.Measurement) (any, error) { + return webconnectivity.GetSummaryKeys(m) +} + +// KibiBytesReceived implements Experiment +func (e *WebConnectivity) KibiBytesReceived() float64 { + return e.byteCounter.KibiBytesReceived() +} + +// KibiBytesSent implements Experiment +func (e *WebConnectivity) KibiBytesSent() float64 { + return e.byteCounter.KibiBytesSent() +} + +// Measure implements Experiment +func (e *WebConnectivity) Measure(ctx context.Context, input string) (*model.Measurement, error) { + // TODO(bassosimone): how to measure the bytes sent and received? + req := &session.WebConnectivityRequest{ + Input: input, + ReportID: e.config.CheckIn.Tests.WebConnectivity.ReportID, + TestStartTime: e.testStartTime, + } + runtimex.Assert(e.config.Session.session != nil, "expected non-nil Session.session") + sess := e.config.Session + if err := sess.session.Send(ctx, &session.Request{WebConnectivity: req}); err != nil { + return nil, err + } + for { + resp, err := sess.session.Recv(ctx) + if err != nil { + return nil, err + } + if resp.Log != nil { + sess.emitLog(resp.Log) + continue + } + if resp.WebConnectivity != nil { + return resp.WebConnectivity.Measurement, resp.WebConnectivity.Error + } + } +} + +// Name implements Experiment +func (e *WebConnectivity) Name() string { + return webconnectivity.ExperimentName +} + +// ReportID implements Experiment +func (e *WebConnectivity) ReportID() string { + return e.config.CheckIn.Tests.WebConnectivity.ReportID +} + +// Session implements Experiment +func (e *WebConnectivity) Session() *Session { + return e.config.Session +} From c06b8856aecb436be4967eebdf510623776766c0 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 8 Feb 2023 23:28:49 +0000 Subject: [PATCH 16/17] poc: show how to improve ooniprobe with ~nice code --- cmd/ooniprobe/internal/cli/geoip/geoip.go | 20 +++---- cmd/ooniprobe/internal/nettests/dash.go | 8 ++- cmd/ooniprobe/internal/nettests/dnscheck.go | 33 +----------- .../internal/nettests/facebook_messenger.go | 10 ++-- .../http_header_field_manipulation.go | 10 ++-- .../nettests/http_invalid_request_line.go | 10 ++-- cmd/ooniprobe/internal/nettests/ndt.go | 9 ++-- cmd/ooniprobe/internal/nettests/nettests.go | 45 +++++++--------- cmd/ooniprobe/internal/nettests/psiphon.go | 10 ++-- cmd/ooniprobe/internal/nettests/riseupvpn.go | 11 ++-- cmd/ooniprobe/internal/nettests/run.go | 35 +++++++----- cmd/ooniprobe/internal/nettests/signal.go | 10 ++-- .../internal/nettests/stunreachability.go | 33 +----------- cmd/ooniprobe/internal/nettests/telegram.go | 10 ++-- cmd/ooniprobe/internal/nettests/tor.go | 10 ++-- cmd/ooniprobe/internal/nettests/torsf.go | 8 ++- cmd/ooniprobe/internal/nettests/vanillator.go | 8 ++- .../internal/nettests/web_connectivity.go | 53 ++++++------------- cmd/ooniprobe/internal/nettests/whatsapp.go | 10 ++-- cmd/ooniprobe/internal/ooni/ooni.go | 20 ++----- internal/nettests/experiment.go | 3 -- internal/nettests/session.go | 11 ++-- internal/nettests/webconnectivity.go | 5 -- 23 files changed, 122 insertions(+), 260 deletions(-) diff --git a/cmd/ooniprobe/internal/cli/geoip/geoip.go b/cmd/ooniprobe/internal/cli/geoip/geoip.go index 21d4ea59d8..27ddf572cb 100644 --- a/cmd/ooniprobe/internal/cli/geoip/geoip.go +++ b/cmd/ooniprobe/internal/cli/geoip/geoip.go @@ -8,7 +8,6 @@ import ( "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/cli/root" "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/ooni" "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/output" - "github.com/ooni/probe-cli/v3/internal/model" ) func init() { @@ -37,23 +36,26 @@ func dogeoip(config dogeoipconfig) error { return err } - engine, err := probeCLI.NewProbeEngine(context.Background(), model.RunTypeManual) - if err != nil { + sess := probeCLI.NewSession(context.Background(), "manual") + defer sess.Close() + + if err := sess.Bootstrap(context.Background()); err != nil { + log.WithError(err).Error("Failed to bootstrap the measurement session") return err } - defer engine.Close() - err = engine.MaybeLookupLocation() + location, err := sess.Geolocate(context.Background()) if err != nil { + log.WithError(err).Error("Failed to lookup the location of the probe") return err } config.Logger.WithFields(log.Fields{ "type": "table", - "asn": engine.ProbeASNString(), - "network_name": engine.ProbeNetworkName(), - "country_code": engine.ProbeCC(), - "ip": engine.ProbeIP(), + "asn": location.ProbeASNString(), + "network_name": location.ProbeNetworkName(), + "country_code": location.ProbeCC(), + "ip": location.ProbeIP(), }).Info("Looked up your location") return nil diff --git a/cmd/ooniprobe/internal/nettests/dash.go b/cmd/ooniprobe/internal/nettests/dash.go index 80f8618dff..e9bcc588ce 100644 --- a/cmd/ooniprobe/internal/nettests/dash.go +++ b/cmd/ooniprobe/internal/nettests/dash.go @@ -1,14 +1,12 @@ package nettests +import "errors" + // Dash test implementation type Dash struct { } // Run starts the test func (d Dash) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder("dash") - if err != nil { - return err - } - return ctl.Run(builder, []string{""}) + return errors.New("not implemented") } diff --git a/cmd/ooniprobe/internal/nettests/dnscheck.go b/cmd/ooniprobe/internal/nettests/dnscheck.go index beebcd306d..e305dad00b 100644 --- a/cmd/ooniprobe/internal/nettests/dnscheck.go +++ b/cmd/ooniprobe/internal/nettests/dnscheck.go @@ -1,42 +1,13 @@ package nettests import ( - "context" - - engine "github.com/ooni/probe-cli/v3/internal/engine" - "github.com/ooni/probe-cli/v3/internal/model" + "errors" ) // DNSCheck nettest implementation. type DNSCheck struct{} -func (n DNSCheck) lookupURLs(ctl *Controller) ([]string, error) { - inputloader := &engine.InputLoader{ - CheckInConfig: &model.OOAPICheckInConfig{ - // not needed because we have default static input in the engine - }, - ExperimentName: "dnscheck", - InputPolicy: model.InputOrStaticDefault, - Session: ctl.Session, - SourceFiles: ctl.InputFiles, - StaticInputs: ctl.Inputs, - } - testlist, err := inputloader.Load(context.Background()) - if err != nil { - return nil, err - } - return ctl.BuildAndSetInputIdxMap(testlist) -} - // Run starts the nettest. func (n DNSCheck) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder("dnscheck") - if err != nil { - return err - } - urls, err := n.lookupURLs(ctl) - if err != nil { - return err - } - return ctl.Run(builder, urls) + return errors.New("not implemented") } diff --git a/cmd/ooniprobe/internal/nettests/facebook_messenger.go b/cmd/ooniprobe/internal/nettests/facebook_messenger.go index 1316babee5..cafaa0f361 100644 --- a/cmd/ooniprobe/internal/nettests/facebook_messenger.go +++ b/cmd/ooniprobe/internal/nettests/facebook_messenger.go @@ -1,16 +1,12 @@ package nettests +import "errors" + // FacebookMessenger test implementation type FacebookMessenger struct { } // Run starts the test func (h FacebookMessenger) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder( - "facebook_messenger", - ) - if err != nil { - return err - } - return ctl.Run(builder, []string{""}) + return errors.New("not implemented") } diff --git a/cmd/ooniprobe/internal/nettests/http_header_field_manipulation.go b/cmd/ooniprobe/internal/nettests/http_header_field_manipulation.go index 6fdd39688f..7cbe17793d 100644 --- a/cmd/ooniprobe/internal/nettests/http_header_field_manipulation.go +++ b/cmd/ooniprobe/internal/nettests/http_header_field_manipulation.go @@ -1,16 +1,12 @@ package nettests +import "errors" + // HTTPHeaderFieldManipulation test implementation type HTTPHeaderFieldManipulation struct { } // Run starts the test func (h HTTPHeaderFieldManipulation) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder( - "http_header_field_manipulation", - ) - if err != nil { - return err - } - return ctl.Run(builder, []string{""}) + return errors.New("not implemented") } diff --git a/cmd/ooniprobe/internal/nettests/http_invalid_request_line.go b/cmd/ooniprobe/internal/nettests/http_invalid_request_line.go index fb87e462d6..654a3cecc9 100644 --- a/cmd/ooniprobe/internal/nettests/http_invalid_request_line.go +++ b/cmd/ooniprobe/internal/nettests/http_invalid_request_line.go @@ -1,16 +1,12 @@ package nettests +import "errors" + // HTTPInvalidRequestLine test implementation type HTTPInvalidRequestLine struct { } // Run starts the test func (h HTTPInvalidRequestLine) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder( - "http_invalid_request_line", - ) - if err != nil { - return err - } - return ctl.Run(builder, []string{""}) + return errors.New("not implemented") } diff --git a/cmd/ooniprobe/internal/nettests/ndt.go b/cmd/ooniprobe/internal/nettests/ndt.go index b8848b0056..1368ee5c47 100644 --- a/cmd/ooniprobe/internal/nettests/ndt.go +++ b/cmd/ooniprobe/internal/nettests/ndt.go @@ -1,15 +1,12 @@ package nettests +import "errors" + // NDT test implementation. We use v7 of NDT since 2020-03-12. type NDT struct { } // Run starts the test func (n NDT) Run(ctl *Controller) error { - // Since 2020-03-18 probe-engine exports v7 as "ndt". - builder, err := ctl.Session.NewExperimentBuilder("ndt") - if err != nil { - return err - } - return ctl.Run(builder, []string{""}) + return errors.New("not implemented") } diff --git a/cmd/ooniprobe/internal/nettests/nettests.go b/cmd/ooniprobe/internal/nettests/nettests.go index aaa502ddce..3f9ea2ddb4 100644 --- a/cmd/ooniprobe/internal/nettests/nettests.go +++ b/cmd/ooniprobe/internal/nettests/nettests.go @@ -11,6 +11,7 @@ import ( "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/ooni" "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/output" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/nettests" "github.com/pkg/errors" ) @@ -21,7 +22,8 @@ type Nettest interface { // NewController creates a nettest controller func NewController( - nt Nettest, probe *ooni.Probe, res *model.DatabaseResult, sess ooni.ProbeEngine) *Controller { + nt Nettest, probe *ooni.Probe, res *model.DatabaseResult, + sess *nettests.Session) *Controller { return &Controller{ Probe: probe, nt: nt, @@ -33,15 +35,16 @@ func NewController( // Controller is passed to the run method of every Nettest // each nettest instance has one controller type Controller struct { - Probe *ooni.Probe - Session ooni.ProbeEngine - res *model.DatabaseResult - nt Nettest - ntCount int - ntIndex int - ntStartTime time.Time // used to calculate the eta - msmts map[int64]*model.DatabaseMeasurement - inputIdxMap map[int64]int64 // Used to map mk idx to database id + CheckInResult *model.OOAPICheckInResult + Probe *ooni.Probe + Session *nettests.Session + res *model.DatabaseResult + nt Nettest + ntCount int + ntIndex int + ntStartTime time.Time // used to calculate the eta + msmts map[int64]*model.DatabaseMeasurement + inputIdxMap map[int64]int64 // Used to map mk idx to database id // InputFiles optionally contains the names of the input // files to read inputs from (only for nettests that take @@ -119,13 +122,12 @@ func (c *Controller) SetNettestIndex(i, n int) { // // This function will continue to run in most cases but will // immediately halt if something's wrong with the file system. -func (c *Controller) Run(builder model.ExperimentBuilder, inputs []string) error { +func (c *Controller) Run(factory nettests.ExperimentFactory, inputs []string) error { db := c.Probe.DB() + c.numInputs = len(inputs) // This will configure the controller as handler for the callbacks // called by ooni/probe-engine/experiment.Experiment. - builder.SetCallbacks(model.ExperimentCallbacks(c)) - c.numInputs = len(inputs) - exp := builder.NewExperiment() + exp := factory.NewExperiment(c) defer func() { c.res.DataUsageDown += exp.KibiBytesReceived() c.res.DataUsageUp += exp.KibiBytesSent() @@ -141,14 +143,7 @@ func (c *Controller) Run(builder model.ExperimentBuilder, inputs []string) error log.Debug(color.RedString("status.started")) if c.Probe.Config().Sharing.UploadResults { - if err := exp.OpenReportContext(context.Background()); err != nil { - log.Debugf( - "%s: %s", color.RedString("failure.report_create"), err.Error(), - ) - } else { - log.Debugf(color.RedString("status.report_create")) - reportID = sql.NullString{String: exp.ReportID(), Valid: true} - } + reportID = sql.NullString{String: exp.ReportID(), Valid: true} } maxRuntime := time.Duration(c.Probe.Config().Nettests.WebsitesMaxRuntime) * time.Second @@ -195,7 +190,7 @@ func (c *Controller) Run(builder model.ExperimentBuilder, inputs []string) error if input != "" { c.OnProgress(0, fmt.Sprintf("processing input: %s", input)) } - measurement, err := exp.MeasureWithContext(context.Background(), input) + measurement, err := exp.Measure(context.Background(), input) if err != nil { log.WithError(err).Debug(color.RedString("failure.measurement")) if err := db.Failed(c.msmts[idx64], err.Error()); err != nil { @@ -216,7 +211,7 @@ func (c *Controller) Run(builder model.ExperimentBuilder, inputs []string) error // Implementation note: SubmitMeasurement will fail here if we did fail // to open the report but we still want to continue. There will be a // bit of a spew in the logs, perhaps, but stopping seems less efficient. - if err := exp.SubmitAndUpdateMeasurementContext(context.Background(), measurement); err != nil { + if err := c.Session.Submit(context.Background(), measurement); err != nil { log.Debug(color.RedString("failure.measurement_submission")) if err := db.UploadFailed(c.msmts[idx64], err.Error()); err != nil { return errors.Wrap(err, "failed to mark upload as failed") @@ -230,7 +225,7 @@ func (c *Controller) Run(builder model.ExperimentBuilder, inputs []string) error } // We only save the measurement to disk if we failed to upload the measurement if saveToDisk { - if err := exp.SaveMeasurement(measurement, msmt.MeasurementFilePath.String); err != nil { + if err := nettests.SaveMeasurement(measurement, msmt.MeasurementFilePath.String); err != nil { return errors.Wrap(err, "failed to save measurement on disk") } } diff --git a/cmd/ooniprobe/internal/nettests/psiphon.go b/cmd/ooniprobe/internal/nettests/psiphon.go index 940340cc4e..44dc51ef52 100644 --- a/cmd/ooniprobe/internal/nettests/psiphon.go +++ b/cmd/ooniprobe/internal/nettests/psiphon.go @@ -1,16 +1,12 @@ package nettests +import "errors" + // Psiphon test implementation type Psiphon struct { } // Run starts the test func (h Psiphon) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder( - "psiphon", - ) - if err != nil { - return err - } - return ctl.Run(builder, []string{""}) + return errors.New("not implemented") } diff --git a/cmd/ooniprobe/internal/nettests/riseupvpn.go b/cmd/ooniprobe/internal/nettests/riseupvpn.go index 185fbcefe4..ccf041f856 100644 --- a/cmd/ooniprobe/internal/nettests/riseupvpn.go +++ b/cmd/ooniprobe/internal/nettests/riseupvpn.go @@ -1,17 +1,12 @@ package nettests +import "errors" + // RiseupVPN test implementation type RiseupVPN struct { } // Run starts the test func (h RiseupVPN) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder( - "riseupvpn", - ) - if err != nil { - return err - } - - return ctl.Run(builder, []string{""}) + return errors.New("not implemented") } diff --git a/cmd/ooniprobe/internal/nettests/run.go b/cmd/ooniprobe/internal/nettests/run.go index 1c6a8371c3..59347fa439 100644 --- a/cmd/ooniprobe/internal/nettests/run.go +++ b/cmd/ooniprobe/internal/nettests/run.go @@ -60,33 +60,42 @@ func RunGroup(config RunGroupConfig) error { return nil } - sess, err := config.Probe.NewProbeEngine(context.Background(), config.RunType) - if err != nil { - log.WithError(err).Error("Failed to create a measurement session") + sess := config.Probe.NewSession(context.Background(), config.RunType) + defer sess.Close() + + if err := sess.Bootstrap(context.Background()); err != nil { + log.WithError(err).Error("Failed to bootstrap the measurement session") return err } - defer sess.Close() - err = sess.MaybeLookupLocation() + location, err := sess.Geolocate(context.Background()) if err != nil { log.WithError(err).Error("Failed to lookup the location of the probe") return err } db := config.Probe.DB() - network, err := db.CreateNetwork(sess) + network, err := db.CreateNetwork(location) if err != nil { log.WithError(err).Error("Failed to create the network row") return err } + + log.Debugf( + "Enabled category codes are the following %v", + config.Probe.Config().Nettests.WebsitesEnabledCategoryCodes, + ) checkInConfig := &model.OOAPICheckInConfig{ + // Setting Charging and OnWiFi to true causes the CheckIn + // API to return to us as much URL as possible with the + // given RunType hint. Charging: true, OnWiFi: true, Platform: platform.Name(), - ProbeASN: sess.ProbeASNString(), - ProbeCC: sess.ProbeCC(), + ProbeASN: location.ProbeASNString(), + ProbeCC: location.ProbeCC(), RunType: config.RunType, - SoftwareName: sess.SoftwareName(), - SoftwareVersion: sess.SoftwareVersion(), + SoftwareName: sess.BootstrapRequest().SoftwareName, + SoftwareVersion: sess.BootstrapRequest().SoftwareVersion, WebConnectivity: model.OOAPICheckInConfigWebConnectivity{ CategoryCodes: config.Probe.Config().Nettests.WebsitesEnabledCategoryCodes, }, @@ -94,8 +103,9 @@ func RunGroup(config RunGroupConfig) error { if checkInConfig.WebConnectivity.CategoryCodes == nil { checkInConfig.WebConnectivity.CategoryCodes = []string{} } - if err := sess.MaybeLookupBackends(checkInConfig); err != nil { - log.WithError(err).Warn("Failed to discover OONI backends") + checkInResult, err := sess.CheckIn(context.Background(), checkInConfig) + if err != nil { + log.WithError(err).Warn("Failed to query the check-in API") return err } @@ -128,6 +138,7 @@ func RunGroup(config RunGroupConfig) error { } log.Debugf("Running test %T", nt) ctl := NewController(nt, config.Probe, result, sess) + ctl.CheckInResult = checkInResult ctl.InputFiles = config.InputFiles ctl.Inputs = config.Inputs ctl.RunType = config.RunType diff --git a/cmd/ooniprobe/internal/nettests/signal.go b/cmd/ooniprobe/internal/nettests/signal.go index 3a2df6b874..481e6df09f 100644 --- a/cmd/ooniprobe/internal/nettests/signal.go +++ b/cmd/ooniprobe/internal/nettests/signal.go @@ -1,15 +1,11 @@ package nettests +import "errors" + // Signal nettest implementation. type Signal struct{} // Run starts the nettest. func (h Signal) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder( - "signal", - ) - if err != nil { - return err - } - return ctl.Run(builder, []string{""}) + return errors.New("not implemented") } diff --git a/cmd/ooniprobe/internal/nettests/stunreachability.go b/cmd/ooniprobe/internal/nettests/stunreachability.go index 186bb7bb98..f8fa58377b 100644 --- a/cmd/ooniprobe/internal/nettests/stunreachability.go +++ b/cmd/ooniprobe/internal/nettests/stunreachability.go @@ -1,42 +1,13 @@ package nettests import ( - "context" - - engine "github.com/ooni/probe-cli/v3/internal/engine" - "github.com/ooni/probe-cli/v3/internal/model" + "errors" ) // STUNReachability nettest implementation. type STUNReachability struct{} -func (n STUNReachability) lookupURLs(ctl *Controller) ([]string, error) { - inputloader := &engine.InputLoader{ - CheckInConfig: &model.OOAPICheckInConfig{ - // not needed because we have default static input in the engine - }, - ExperimentName: "stunreachability", - InputPolicy: model.InputOrStaticDefault, - Session: ctl.Session, - SourceFiles: ctl.InputFiles, - StaticInputs: ctl.Inputs, - } - testlist, err := inputloader.Load(context.Background()) - if err != nil { - return nil, err - } - return ctl.BuildAndSetInputIdxMap(testlist) -} - // Run starts the nettest. func (n STUNReachability) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder("stunreachability") - if err != nil { - return err - } - urls, err := n.lookupURLs(ctl) - if err != nil { - return err - } - return ctl.Run(builder, urls) + return errors.New("not implemented") } diff --git a/cmd/ooniprobe/internal/nettests/telegram.go b/cmd/ooniprobe/internal/nettests/telegram.go index 82d75d82c2..3f4ccd48c5 100644 --- a/cmd/ooniprobe/internal/nettests/telegram.go +++ b/cmd/ooniprobe/internal/nettests/telegram.go @@ -1,16 +1,12 @@ package nettests +import "errors" + // Telegram test implementation type Telegram struct { } // Run starts the test func (h Telegram) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder( - "telegram", - ) - if err != nil { - return err - } - return ctl.Run(builder, []string{""}) + return errors.New("not implemented") } diff --git a/cmd/ooniprobe/internal/nettests/tor.go b/cmd/ooniprobe/internal/nettests/tor.go index 96bfbb7d2f..5949136815 100644 --- a/cmd/ooniprobe/internal/nettests/tor.go +++ b/cmd/ooniprobe/internal/nettests/tor.go @@ -1,16 +1,12 @@ package nettests +import "errors" + // Tor test implementation type Tor struct { } // Run starts the test func (h Tor) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder( - "tor", - ) - if err != nil { - return err - } - return ctl.Run(builder, []string{""}) + return errors.New("not implemented") } diff --git a/cmd/ooniprobe/internal/nettests/torsf.go b/cmd/ooniprobe/internal/nettests/torsf.go index 494dae4743..8072e93ada 100644 --- a/cmd/ooniprobe/internal/nettests/torsf.go +++ b/cmd/ooniprobe/internal/nettests/torsf.go @@ -1,16 +1,14 @@ package nettests +import "errors" + // TorSf test implementation type TorSf struct { } // Run starts the test func (h TorSf) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder("torsf") - if err != nil { - return err - } - return ctl.Run(builder, []string{""}) + return errors.New("not implemented") } func (h TorSf) onlyBackground() {} diff --git a/cmd/ooniprobe/internal/nettests/vanillator.go b/cmd/ooniprobe/internal/nettests/vanillator.go index 11d7266549..e6b339dea1 100644 --- a/cmd/ooniprobe/internal/nettests/vanillator.go +++ b/cmd/ooniprobe/internal/nettests/vanillator.go @@ -1,16 +1,14 @@ package nettests +import "errors" + // VanillaTor test implementation type VanillaTor struct { } // Run starts the test func (h VanillaTor) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder("vanilla_tor") - if err != nil { - return err - } - return ctl.Run(builder, []string{""}) + return errors.New("not implemented") } func (h VanillaTor) onlyBackground() {} diff --git a/cmd/ooniprobe/internal/nettests/web_connectivity.go b/cmd/ooniprobe/internal/nettests/web_connectivity.go index 6b453f2793..efbbfc6a66 100644 --- a/cmd/ooniprobe/internal/nettests/web_connectivity.go +++ b/cmd/ooniprobe/internal/nettests/web_connectivity.go @@ -1,42 +1,14 @@ package nettests import ( - "context" - - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/config" - engine "github.com/ooni/probe-cli/v3/internal/engine" - "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/nettests" ) -func newCheckInConfig(runType model.RunType, config *config.Config) *model.OOAPICheckInConfig { - result := &model.OOAPICheckInConfig{ - // Setting Charging and OnWiFi to true causes the CheckIn - // API to return to us as much URL as possible with the - // given RunType hint. - Charging: true, - OnWiFi: true, - RunType: runType, - WebConnectivity: model.OOAPICheckInConfigWebConnectivity{ - CategoryCodes: config.Nettests.WebsitesEnabledCategoryCodes, - }, - } - if result.WebConnectivity.CategoryCodes == nil { - result.WebConnectivity.CategoryCodes = []string{} - } - return result -} - -func (n WebConnectivity) lookupURLs(ctl *Controller, config *config.Config) ([]string, error) { - inputloader := &engine.InputLoader{ - CheckInConfig: newCheckInConfig(ctl.RunType, config), - ExperimentName: "web_connectivity", - InputPolicy: model.InputOrQueryBackend, - Session: ctl.Session, - SourceFiles: ctl.InputFiles, - StaticInputs: ctl.Inputs, - } - testlist, err := inputloader.Load(context.Background()) +func (n WebConnectivity) lookupURLs( + ctl *Controller, + factory *nettests.WebConnectivityFactory, +) ([]string, error) { + testlist, err := factory.LoadInputs() if err != nil { return nil, err } @@ -48,14 +20,19 @@ type WebConnectivity struct{} // Run starts the test func (n WebConnectivity) Run(ctl *Controller) error { - log.Debugf("Enabled category codes are the following %v", ctl.Probe.Config().Nettests.WebsitesEnabledCategoryCodes) - urls, err := n.lookupURLs(ctl, ctl.Probe.Config()) + factoryConfig := &nettests.WebConnectivityFactoryConfig{ + CheckIn: ctl.CheckInResult, + InputFiles: ctl.InputFiles, + Inputs: ctl.Inputs, + Session: ctl.Session, + } + factory, err := nettests.NewWebConnectivityFactory(factoryConfig) if err != nil { return err } - builder, err := ctl.Session.NewExperimentBuilder("web_connectivity") + urls, err := n.lookupURLs(ctl, factory) if err != nil { return err } - return ctl.Run(builder, urls) + return ctl.Run(factory, urls) } diff --git a/cmd/ooniprobe/internal/nettests/whatsapp.go b/cmd/ooniprobe/internal/nettests/whatsapp.go index 4660abe006..bc8e3523eb 100644 --- a/cmd/ooniprobe/internal/nettests/whatsapp.go +++ b/cmd/ooniprobe/internal/nettests/whatsapp.go @@ -1,16 +1,12 @@ package nettests +import "errors" + // WhatsApp test implementation type WhatsApp struct { } // Run starts the test func (h WhatsApp) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder( - "whatsapp", - ) - if err != nil { - return err - } - return ctl.Run(builder, []string{""}) + return errors.New("not implemented") } diff --git a/cmd/ooniprobe/internal/ooni/ooni.go b/cmd/ooniprobe/internal/ooni/ooni.go index c955b68683..9e4b1e1e8a 100644 --- a/cmd/ooniprobe/internal/ooni/ooni.go +++ b/cmd/ooniprobe/internal/ooni/ooni.go @@ -17,6 +17,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/legacy/assetsdir" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/nettests" "github.com/ooni/probe-cli/v3/internal/session" "github.com/pkg/errors" ) @@ -36,7 +37,7 @@ type ProbeCLI interface { IsBatch() bool Home() string TempDir() string - NewProbeEngine(ctx context.Context, runType model.RunType) (ProbeEngine, error) + NewSession(ctx context.Context, runType model.RunType) *nettests.Session } // ProbeEngine is an instance of the OONI Probe engine. @@ -213,10 +214,8 @@ func (p *Probe) Init(softwareName, softwareVersion, proxy string) error { return nil } -// newSession creates a new ooni/probe-engine session using the -// current configuration inside the context. The caller must close -// the session when done using it, by calling sess.Close(). -func (p *Probe) newSession(ctx context.Context, runType model.RunType) (*engineSession, error) { +// NewSession creates a new measurement session. +func (p *Probe) NewSession(ctx context.Context, runType model.RunType) *nettests.Session { // When the software name is the default software name and we're running // in unattended mode, adjust the software name accordingly. @@ -252,16 +251,7 @@ func (p *Probe) newSession(ctx context.Context, runType model.RunType) (*engineS VerboseLogging: false, } - return newSession(config, logger), nil -} - -// NewProbeEngine creates a new ProbeEngine instance. -func (p *Probe) NewProbeEngine(ctx context.Context, runType model.RunType) (ProbeEngine, error) { - sess, err := p.newSession(ctx, runType) - if err != nil { - return nil, err - } - return sess, nil + return nettests.NewSession(config, logger) } // NewProbe creates a new probe instance. diff --git a/internal/nettests/experiment.go b/internal/nettests/experiment.go index 7db1011144..7ca9c776ff 100644 --- a/internal/nettests/experiment.go +++ b/internal/nettests/experiment.go @@ -34,9 +34,6 @@ type Experiment interface { // ReportID returns the reportID used by this experiment. ReportID() string - - // Session returns the session owning this experiment. - Session() *Session } // ErrMissingCheckInConfig is the error returned when we do not have diff --git a/internal/nettests/session.go b/internal/nettests/session.go index 812194caa3..d166b98f36 100644 --- a/internal/nettests/session.go +++ b/internal/nettests/session.go @@ -42,6 +42,11 @@ func (s *Session) Close() error { return nil } +// BootstrapRequest returns the configured bootstrap request. +func (s *Session) BootstrapRequest() *session.BootstrapRequest { + return s.bootstrapRequest +} + // Bootstrap performs the session bootstrap. func (s *Session) Bootstrap(ctx context.Context) error { if err := s.session.Send(ctx, &session.Request{Bootstrap: s.bootstrapRequest}); err != nil { @@ -64,9 +69,6 @@ func (s *Session) Bootstrap(ctx context.Context) error { // Geolocate runs the geolocate task. func (s *Session) Geolocate(ctx context.Context) (*geolocate.Results, error) { - if err := s.Bootstrap(ctx); err != nil { - return nil, err - } req := &session.GeolocateRequest{} if err := s.session.Send(ctx, &session.Request{Geolocate: req}); err != nil { return nil, err @@ -89,9 +91,6 @@ func (s *Session) Geolocate(ctx context.Context) (*geolocate.Results, error) { // CheckIn runs the checkIn task. func (s *Session) CheckIn( ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResult, error) { - if err := s.Bootstrap(ctx); err != nil { - return nil, err - } if err := s.session.Send(ctx, &session.Request{CheckIn: config}); err != nil { return nil, err } diff --git a/internal/nettests/webconnectivity.go b/internal/nettests/webconnectivity.go index b830895ac9..81d0130a5e 100644 --- a/internal/nettests/webconnectivity.go +++ b/internal/nettests/webconnectivity.go @@ -139,8 +139,3 @@ func (e *WebConnectivity) Name() string { func (e *WebConnectivity) ReportID() string { return e.config.CheckIn.Tests.WebConnectivity.ReportID } - -// Session implements Experiment -func (e *WebConnectivity) Session() *Session { - return e.config.Session -} From b2b1f06b8b505dd8cd3d91f40f54d0c4090d5887 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 17 Feb 2023 08:37:32 +0100 Subject: [PATCH 17/17] more experimental code --- internal/cmd/tortunnel/main.go | 20 ++ internal/model/location.go | 30 +++ internal/model/tunnel.go | 47 +++++ internal/optional/optional.go | 101 ++++++++++ internal/optional/optional_test.go | 265 +++++++++++++++++++++++++ internal/shellx/shellx.go | 6 +- internal/tortunnel-old/config.go | 81 ++++++++ internal/tortunnel-old/dependencies.go | 39 ++++ internal/tortunnel-old/desktop.go | 41 ++++ internal/tortunnel-old/doc.go | 2 + internal/tortunnel-old/start.go | 165 +++++++++++++++ internal/tortunnel-old/tunnel.go | 79 ++++++++ internal/tortunnel/config.go | 85 ++++++++ internal/tortunnel/dependencies.go | 43 ++++ internal/tortunnel/doc.go | 2 + internal/tortunnel/tunnel.go | 7 + 16 files changed, 1010 insertions(+), 3 deletions(-) create mode 100644 internal/cmd/tortunnel/main.go create mode 100644 internal/model/tunnel.go create mode 100644 internal/optional/optional.go create mode 100644 internal/optional/optional_test.go create mode 100644 internal/tortunnel-old/config.go create mode 100644 internal/tortunnel-old/dependencies.go create mode 100644 internal/tortunnel-old/desktop.go create mode 100644 internal/tortunnel-old/doc.go create mode 100644 internal/tortunnel-old/start.go create mode 100644 internal/tortunnel-old/tunnel.go create mode 100644 internal/tortunnel/config.go create mode 100644 internal/tortunnel/dependencies.go create mode 100644 internal/tortunnel/doc.go create mode 100644 internal/tortunnel/tunnel.go diff --git a/internal/cmd/tortunnel/main.go b/internal/cmd/tortunnel/main.go new file mode 100644 index 0000000000..f6c5d59148 --- /dev/null +++ b/internal/cmd/tortunnel/main.go @@ -0,0 +1,20 @@ +package main + +import ( + "context" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/tortunnel" +) + +func main() { + config := &tortunnel.Config{ + Logger: log.Log, + } + ctx := context.Background() + tunnel, err := tortunnel.Start(ctx, config) + if err != nil { + log.Fatalf("failure: %s", err.Error()) + } + tunnel.Stop() +} diff --git a/internal/model/location.go b/internal/model/location.go index fad7e0cb6d..6c1735ae63 100644 --- a/internal/model/location.go +++ b/internal/model/location.go @@ -29,3 +29,33 @@ type LocationProvider interface { // ResolverNetworkName is the name of the ResolverASN. ResolverNetworkName() string } + +// Location describes the probe's location. +type Location struct { + // ProbeASN is the ASN associated with ProbeIP. + ProbeASN int64 + + // ProbeASNString returns the probe ASN as the AS%d string. + ProbeASNString string + + // ProbeCC is the country code associated with ProbeIP. + ProbeCC string + + // ProbeIP is the probe IP address. + ProbeIP string + + // ProbeNetworkName is the name of the ProbeASN. + ProbeNetworkName string + + // ResolverIP is the IP of the resolver. + ResolverIP string + + // ResolverASN is the resolver ASN. + ResolverASN int64 + + // ResolverASNString is the resolver ASN as the AS%d string. + ResolverASNString string + + // ResolverNetworkName is the name of the ResolverASN. + ResolverNetworkName string +} diff --git a/internal/model/tunnel.go b/internal/model/tunnel.go new file mode 100644 index 0000000000..25d5def495 --- /dev/null +++ b/internal/model/tunnel.go @@ -0,0 +1,47 @@ +package model + +// +// Common interface for circumvention tunnels. +// + +import ( + "context" + "errors" +) + +// TunnelBootstrapEvent is an event emitted during the bootstrap. +type TunnelBootstrapEvent struct { + // Progress is the progress we have made so far as a number between 0 and 1. + Progress float64 + + // Message is the corresponding explanatory message. + Message string +} + +// Tunnel is a tunnel for communicating with the OONI backend. +type Tunnel interface { + // LookupProbeIP discovers the probe's IP address using this tunnel. + LookupProbeIP(ctx context.Context) (string, error) + + // Name returns the tunnel name. + Name() string + + // NewHTTPTransport returns a new HTTP transport using this tunnel. + NewHTTPTransport(logger Logger) (HTTPTransport, error) + + // NewDNSOverHTTPSResolver returns a new DNS-over-HTTPS resolver using this tunnel. + NewDNSOverHTTPSResolver(logger Logger, URL string) (Resolver, error) + + // Start starts the tunnel and returns two channels. The first channel gets + // interim bootstrap events, the second gets the final result. If the context + // is cancelled or expires during the bootstrap, we interrupt the bootstrap + // early and return an error via the error channel. If you already started + // the tunnel, this function posts a nil error on the second channel. + Start(ctx context.Context) (<-chan *TunnelBootstrapEvent, <-chan error) + + // Stop stops the tunnel. + Stop() +} + +// ErrTunnelNotStarted indicates we have not started the tunnel. +var ErrTunnelNotStarted = errors.New("tunnel: not started") diff --git a/internal/optional/optional.go b/internal/optional/optional.go new file mode 100644 index 0000000000..7fcf4b0389 --- /dev/null +++ b/internal/optional/optional.go @@ -0,0 +1,101 @@ +// Package optional contains safer code to handle optional values. +package optional + +import ( + "bytes" + "encoding/json" + "reflect" + + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// Value is an optional value. The zero value of this structure +// is equivalent to the one you get when calling [None]. +type Value[T any] struct { + // indirect is the indirect pointer to the value. + indirect *T +} + +// None constructs an empty value. +func None[T any]() Value[T] { + return Value[T]{nil} +} + +// Some constructs a some value unless T is a pointer and points to +// nil, in which case [Some] is equivalent to [None]. +func Some[T any](value T) Value[T] { + v := Value[T]{} + maybeSetFromValue(&v, value) + return v +} + +// maybeSetFromValue sets the underlying value unless T is a pointer +// in which case we set the Value to be empty. +func maybeSetFromValue[T any](v *Value[T], value T) { + rv := reflect.ValueOf(value) + if rv.Type().Kind() == reflect.Pointer && rv.IsZero() { + v.indirect = nil + return + } + v.indirect = &value +} + +var _ json.Unmarshaler = &Value[int]{} + +// UnmarshalJSON implements json.Unmarshaler. Note that a `null` JSON +// value always leads to an empty Value. +func (v *Value[T]) UnmarshalJSON(data []byte) error { + // A `null` underlying value should always be equivalent to + // invoking the None constructor of for T. While this is not + // what the [json] package recommends doing for this case, + // it is consistent with initializing an optional. + if bytes.Equal(data, []byte(`null`)) { + v.indirect = nil + return nil + } + + // Otherwise, let's try to unmarshal into a real value + var value T + if err := json.Unmarshal(data, &value); err != nil { + return err + } + + // Enforce the same semantics of the Some constructor: treat + // pointer types specially to avoid the case where we have + // a Value that is wrapping a nil pointer but for which the + // IsNone check actually returns false. (Maybe this check is + // redundant but it seems better to enforce it anyway.) + maybeSetFromValue(v, value) + return nil +} + +var _ json.Marshaler = Value[int]{} + +// MarshalJSON implements json.Marshaler. An empty value serializes +// to `null` and otherwise we serialize the underluing value. +func (v Value[T]) MarshalJSON() ([]byte, error) { + if v.indirect == nil { + return json.Marshal(nil) + } + return json.Marshal(*v.indirect) +} + +// IsNone returns whether this [Value] is empty. +func (v Value[T]) IsNone() bool { + return v.indirect == nil +} + +// Unwrap returns the underlying value or panics. In case of +// panic, the value passed to panic is an error. +func (v Value[T]) Unwrap() T { + runtimex.Assert(!v.IsNone(), "is none") + return *v.indirect +} + +// UnwrapOr returns the fallback if the [Value] is empty. +func (v Value[T]) UnwrapOr(fallback T) T { + if v.IsNone() { + return fallback + } + return v.Unwrap() +} diff --git a/internal/optional/optional_test.go b/internal/optional/optional_test.go new file mode 100644 index 0000000000..dff22a0b0a --- /dev/null +++ b/internal/optional/optional_test.go @@ -0,0 +1,265 @@ +package optional + +import ( + "encoding/json" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestValue(t *testing.T) { + + // Verify that None creates a Value with an indirect == nil + t.Run("None works as intended", func(t *testing.T) { + v := None[int]() + if v.indirect != nil { + t.Fatal("should be nil") + } + }) + + t.Run("Some works as intended", func(t *testing.T) { + + // Verify that Some(value) creates a valid underlying pointer to + // the value when the wrapped type is not a pointer. + t.Run("for nonzero nonpointer value", func(t *testing.T) { + underlying := 12345 + v := Some(underlying) + if v.indirect == nil || *v.indirect != underlying { + t.Fatal("unexpected indirect") + } + }) + + // Verify that Some(value) works for a zero input when the + // wrapped value is not a pointer. + t.Run("for zero nonpointer value", func(t *testing.T) { + underlying := 0 + v := Some(underlying) + if v.indirect == nil || *v.indirect != underlying { + t.Fatal("unexpected indirect") + } + }) + + // Verify that Some(value) correctly creates a pointer to the + // underlying value when we're wrapping a pointer type + t.Run("for nonzero pointer value", func(t *testing.T) { + underlying := 12345 + v := Some(&underlying) + if v.indirect == nil || *v.indirect == nil || **v.indirect != underlying { + t.Fatal("unexpected indirect") + } + }) + + // Verify that Some(nil) creates an empty value when wrapping a pointer + t.Run("for zero nonpointer value", func(t *testing.T) { + var underlying *int + v := Some(underlying) + if v.indirect != nil { + t.Fatal("unexpected indirect", *v.indirect) + } + }) + }) + + t.Run("UnmarshalJSON works as intended", func(t *testing.T) { + + t.Run("for nonpointer type", func(t *testing.T) { + + // When we wrap a nonpointer and the JSON is valid, we expect + // the underlying value to be correctly populated + t.Run("with valid JSON input", func(t *testing.T) { + type config struct { + UID Value[int64] + } + + input := []byte(`{"UID":12345}`) + var state config + if err := json.Unmarshal(input, &state); err != nil { + t.Fatal(err) + } + + if state.UID.indirect == nil || *state.UID.indirect != 12345 { + t.Fatal("did not set indirect correctly") + } + }) + + // When the JSON input is incompatible, there should always + // be an error indicating we cannot assign and obviously the + // Value should not have been set. + t.Run("with incompatible JSON input", func(t *testing.T) { + type config struct { + UID Value[int64] + } + + input := []byte(`{"UID":[]}`) + var state config + err := json.Unmarshal(input, &state) + if err == nil || err.Error() != "json: cannot unmarshal array into Go struct field config.UID of type int64" { + t.Fatal("unexpected err", err) + } + + if state.UID.indirect != nil { + t.Fatal("should not have set", *state.UID.indirect) + } + }) + + // As a special case, when the JSON input is `null`, we should behave + // like the None constructor had been called. + t.Run("with null JSON input", func(t *testing.T) { + type config struct { + UID Value[int64] + } + + input := []byte(`{"UID":null}`) + var state config + err := json.Unmarshal(input, &state) + if err != nil { + t.Fatal(err) + } + + if state.UID.indirect != nil { + t.Fatal("should not have set", *state.UID.indirect) + } + }) + }) + + t.Run("for pointer type", func(t *testing.T) { + + // When the JSON input is valid, we expect that the underlying pointer + // is a pointer to the expected value. + t.Run("with valid JSON input", func(t *testing.T) { + type config struct { + UID Value[*int64] + } + + input := []byte(`{"UID":12345}`) + var state config + if err := json.Unmarshal(input, &state); err != nil { + t.Fatal(err) + } + + if state.UID.indirect == nil || *state.UID.indirect == nil || **state.UID.indirect != 12345 { + t.Fatal("did not set indirect correctly") + } + }) + + // With incompatible JSON input, there should be an error and obviously + // we should not have set any value inside the Value + t.Run("with incompatible JSON input", func(t *testing.T) { + type config struct { + UID Value[*int64] + } + + input := []byte(`{"UID":[]}`) + var state config + err := json.Unmarshal(input, &state) + if err == nil || err.Error() != "json: cannot unmarshal array into Go struct field config.UID of type int64" { + t.Fatal("unexpected err", err) + } + + if state.UID.indirect != nil { + t.Fatal("should not have set", *state.UID.indirect) + } + }) + + // When the JSON input is `null`, the code should behave like we + // had invoked the None constructor for the pointer type. + t.Run("with null JSON input", func(t *testing.T) { + type config struct { + UID Value[*int64] + } + + input := []byte(`{"UID":null}`) + var state config + err := json.Unmarshal(input, &state) + if err != nil { + t.Fatal(err) + } + + if state.UID.indirect != nil { + t.Fatal("should not have set", *state.UID.indirect) + } + }) + }) + }) + + t.Run("MarshalJSON works as intended", func(t *testing.T) { + t.Run("for an empty Value", func(t *testing.T) { + value := None[int]() + got, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + expect := []byte(`null`) + if diff := cmp.Diff(expect, got); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("for an nonempty Value", func(t *testing.T) { + value := Some(12345) + got, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + expect := []byte(`12345`) + if diff := cmp.Diff(expect, got); diff != "" { + t.Fatal(diff) + } + }) + }) + + t.Run("IsNone works as intended", func(t *testing.T) { + t.Run("for empty Value", func(t *testing.T) { + value := None[int]() + if !value.IsNone() { + t.Fatal("should be none") + } + }) + + t.Run("for nonempty Value", func(t *testing.T) { + value := Some(12345) + if value.IsNone() { + t.Fatal("should not be none") + } + }) + }) + + t.Run("Unwrap works as intended", func(t *testing.T) { + t.Run("for an empty Value", func(t *testing.T) { + value := None[int]() + var err error + func() { + defer func() { + err = recover().(error) + }() + out := value.Unwrap() + t.Log(out) + }() + if err == nil || err.Error() != "is none" { + t.Fatal("unexpected err", err) + } + }) + + t.Run("for a nonempty Value", func(t *testing.T) { + value := Some(12345) + if v := value.Unwrap(); v != 12345 { + t.Fatal("unexpected value", v) + } + }) + }) + + t.Run("UnwrapOr works as intended", func(t *testing.T) { + t.Run("for an empty Value", func(t *testing.T) { + value := None[int]() + if v := value.UnwrapOr(555); v != 555 { + t.Fatal("unexpected value", v) + } + }) + + t.Run("for a nonempty Value", func(t *testing.T) { + value := Some(12345) + if v := value.UnwrapOr(555); v != 12345 { + t.Fatal("unexpected value", v) + } + }) + }) +} diff --git a/internal/shellx/shellx.go b/internal/shellx/shellx.go index 8d09fbf62b..e47ad4c7fe 100644 --- a/internal/shellx/shellx.go +++ b/internal/shellx/shellx.go @@ -138,7 +138,7 @@ func cmd(config *Config, argv *Argv, envp *Envp) *execabs.Cmd { cmd.Env = append(cmd.Env, entry) } if config.Logger != nil { - cmdline := quotedCommandLineUnsafe(argv.P, argv.V...) + cmdline := QuotedCommandLineUnsafe(argv.P, argv.V...) config.Logger.Infof("+ %s", cmdline) } return cmd @@ -271,9 +271,9 @@ func OutputCommandLine(logger model.Logger, cmdline string) ([]byte, error) { // ErrNoCommandToExecute means that the command line is empty. var ErrNoCommandToExecute = errors.New("shellx: no command to execute") -// quotedCommandLineUnsafe returns a quoted command line. This function is unsafe +// QuotedCommandLineUnsafe returns a quoted command line. This function is unsafe // and SHOULD only be used to produce a nice output. -func quotedCommandLineUnsafe(command string, args ...string) string { +func QuotedCommandLineUnsafe(command string, args ...string) string { v := []string{} v = append(v, maybeQuoteArgUnsafe(command)) for _, a := range args { diff --git a/internal/tortunnel-old/config.go b/internal/tortunnel-old/config.go new file mode 100644 index 0000000000..456e842fb1 --- /dev/null +++ b/internal/tortunnel-old/config.go @@ -0,0 +1,81 @@ +package tortunnel + +import ( + "os" + + "github.com/ooni/probe-cli/v3/internal/model" +) + +// Config contains config for starting the tor tunnel. +type Config struct { + // BootstrapEvents is the OPTIONAL channel where to send + // events emitted during the bootstrap. + BootstrapEvents chan<- string `json:",omitempty"` + + // Dependencies is OPTIONAL and allow one to mock the functions + // called by Start, which is mainly useful for testing. + Dependencies *Dependencies `json:",omitempty"` + + // Logger is the OPTIONAL logger to use during the bootstrap. + Logger model.Logger `json:",omitempty"` + + // SnowflakeEnabled OPTIONALLY enables snowflake. + SnowflakeEnabled bool + + // SnowflakeRendezvousMethod is the OPTIONAL snowflake rendezvous method. + SnowflakeRendezvousMethod string + + // TunnelDir is the OPTIONAL directory in which to store state. + TunnelDir string + + // TorArgs contains OPTIONAL arguments to pass to tor. + TorArgs []string + + // TorBinary OPTIONAL tor binary path. + TorBinary string + + // TorVersion is the OPTIONAL channel where we send the version + // of tor that we are attempting to bootstrap. + TorVersion chan<- string `json:",omitempty"` +} + +// logger always returns a valid instance of [model.Logger]. +func (cc *Config) logger() model.Logger { + if cc.Logger == nil { + return model.DiscardLogger + } + return cc.Logger +} + +// dependencies always returns a valid instance of [Dependencies]. +func (cc *Config) dependencies() *Dependencies { + if cc.Dependencies == nil { + return defaultDependencies + } + return cc.Dependencies +} + +// tunnelDir returns the tunnel dir to use or an error. +func (cc *Config) tunnelDir(logger model.Logger) (string, func(), error) { + if cc.TunnelDir == "" { + dir, err := os.MkdirTemp("", "") + if err != nil { + return "", nil, err + } + return dir, cleanupTunnelDir(logger, dir), nil + } + return cc.TunnelDir, func() {}, nil +} + +// cleanupTunnelDir returns a function that removes a temporary tunnel dir. +func cleanupTunnelDir(logger model.Logger, dir string) func() { + return func() { + maybeRemoveDir(logger, dir) + } +} + +// maybeRemoveDir removes a directory if needed. +func maybeRemoveDir(logger model.Logger, dir string) { + logger.Infof("rm -rf %s", dir) + os.RemoveAll(dir) +} diff --git a/internal/tortunnel-old/dependencies.go b/internal/tortunnel-old/dependencies.go new file mode 100644 index 0000000000..1abf2d488d --- /dev/null +++ b/internal/tortunnel-old/dependencies.go @@ -0,0 +1,39 @@ +package tortunnel + +import ( + "context" + + "github.com/cretz/bine/control" + "github.com/cretz/bine/tor" +) + +// Dependencies contains dependencies allowing to mock [Start] for testing. +type Dependencies struct { + // Start should be equivalent to [tor.Start]. + Start func(ctx context.Context, conf *tor.StartConf) (*tor.Tor, error) + + // TorControlProtocolInfo should be equivalent to calling the + // ProtocolInfo method of instance.Control. + TorControlProtocolInfo func(instance *tor.Tor) (*control.ProtocolInfo, error) + + // TorControlGetInfo should be equivalent to calling the GetInfo + // method of the instance.Control. + TorControlGetInfo func(instance *tor.Tor, keys ...string) ([]*control.KeyVal, error) + + // TorEnableNetwork should be equivalent to calling the + // EnableNetwork method of instance. + TorEnableNetwork func(ctx context.Context, instance *tor.Tor, wait bool) error +} + +var defaultDependencies = &Dependencies{ + Start: tor.Start, + TorControlProtocolInfo: func(instance *tor.Tor) (*control.ProtocolInfo, error) { + return instance.Control.ProtocolInfo() + }, + TorControlGetInfo: func(instance *tor.Tor, keys ...string) ([]*control.KeyVal, error) { + return instance.Control.GetInfo(keys...) + }, + TorEnableNetwork: func(ctx context.Context, instance *tor.Tor, wait bool) error { + return instance.EnableNetwork(ctx, wait) + }, +} diff --git a/internal/tortunnel-old/desktop.go b/internal/tortunnel-old/desktop.go new file mode 100644 index 0000000000..fb556008f2 --- /dev/null +++ b/internal/tortunnel-old/desktop.go @@ -0,0 +1,41 @@ +//go:build !android && !ios && !ooni_libtor + +package tortunnel + +// +// This file implements our strategy for running tor on desktop in most +// configurations except for the ooni_libtor case, where we build tor and +// its dependencies for Linux. The purpuse of this special case it that +// of testing the otherwise untested code that would run on Android. +// + +import ( + "github.com/cretz/bine/tor" + "golang.org/x/sys/execabs" +) + +// newTorStartConf in this configuration uses torExePath to get a +// suitable tor binary and then executes it. +func newTorStartConf(config *Config, dataDir string, extraArgs []string) (*tor.StartConf, error) { + // determine the logget to use. + logger := config.logger() + + // determine the tor binary to use. + torBinary := config.TorBinary + if torBinary == "" { + var err error + torBinary, err = execabs.LookPath("tor") + if err != nil { + return nil, err + } + } + logger.Infof("tortunnel: using this tor binary: %s", torBinary) + + // generate and return the config. + tsc := &tor.StartConf{ + ExePath: torBinary, + DataDir: dataDir, + ExtraArgs: extraArgs, + } + return tsc, nil +} diff --git a/internal/tortunnel-old/doc.go b/internal/tortunnel-old/doc.go new file mode 100644 index 0000000000..6fde11a5c8 --- /dev/null +++ b/internal/tortunnel-old/doc.go @@ -0,0 +1,2 @@ +// Package tortunnel implements [model.Tunnel] using tor. +package tortunnel diff --git a/internal/tortunnel-old/start.go b/internal/tortunnel-old/start.go new file mode 100644 index 0000000000..7275a1883e --- /dev/null +++ b/internal/tortunnel-old/start.go @@ -0,0 +1,165 @@ +package tortunnel + +import ( + "context" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/cretz/bine/tor" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/shellx" +) + +// Start attempts to start a tor [Tunnel]. +func Start(ctx context.Context, config *Config) (*Tunnel, error) { + // obtain the logger to use + logger := config.logger() + + // determine the tunnel directory to use + tunnelDir, cleanupTunnelDir, err := config.tunnelDir(logger) + if err != nil { + return nil, err + } + stateDir := filepath.Join(tunnelDir, "tor") + logger.Infof("tortunnel: stateDir: %s", stateDir) + + // determine the log file to use + logFile := filepath.Join(stateDir, "tor.log") + logger.Infof("tortunnel: logFile: %s", logFile) + + // cleanup any leftovers from a previous invocation, if needed + maybeCleanupStateDir(logger, stateDir, logFile) + + // setup command line arguments. + extraArgs := append([]string{}, config.TorArgs...) + extraArgs = append(extraArgs, "Log") + extraArgs = append(extraArgs, fmt.Sprintf(`notice file %s`, logFile)) + logCommandLineArguments(logger, extraArgs) + + // TODO(bassosimone): handle snowflake + + // generate the start configuration + torStartConf, err := newTorStartConf(config, stateDir, extraArgs) + if err != nil { + cleanupTunnelDir() + return nil, err + } + + // obtain the dependencies we should use. + deps := config.dependencies() + + // start the tor process + instance, err := deps.Start(ctx, torStartConf) + if err != nil { + cleanupTunnelDir() + return nil, err + } + + // make sure we close the running process on Close + instance.StopProcessOnClose = true + + // obtain and emit the tor version + protoInfo, err := deps.TorControlProtocolInfo(instance) + if err != nil { + instance.Close() + cleanupTunnelDir() + return nil, err + } + logger.Infof("tortunnel: tor version: %s", protoInfo.TorVersion) + select { + case config.TorVersion <- protoInfo.TorVersion: + default: + } + + // wait for the bootstrap to complete + startBootstrap := time.Now() + if err := deps.TorEnableNetwork(ctx, instance, true); err != nil { + instance.Close() + cleanupTunnelDir() + return nil, err + } + bootstrapTime := time.Since(startBootstrap) + logger.Infof("tortunnel: bootstrap time: %v", bootstrapTime) + + // get the proxy URL + proxyURL, err := getProxyURL(deps, instance) + if err != nil { + instance.Close() + cleanupTunnelDir() + return nil, err + } + + // TODO(bassosimone): we still need to set the name correctly + + // construct a tunnel instance + tunnel := &Tunnel{ + bootstrapTime: bootstrapTime, + instance: instance, + maybeDeleteTunnelDir: cleanupTunnelDir, + name: "", + proxy: proxyURL, + stopOnce: sync.Once{}, + } + + return tunnel, nil +} + +// bootstrap performs the bootstrap and returns its duration. +func bootstrap(ctx context.Context, config *Config, instance *tor.Tor) (time.Duration, error) { +} + +// ErrCannotGetSOCKS5ProxyURL indicates we cannot get the SOCKS5 proxy URL. +var ErrCannotGetSOCKS5ProxyURL = errors.New("tortunnel: cannot get SOCKS5 proxy URL") + +// getProxyURL attempts to obtain the proxy URL. +func getProxyURL(deps *Dependencies, instance *tor.Tor) (*url.URL, error) { + // Adapted from + info, err := deps.TorControlGetInfo(instance, "net/listeners/socks") + if err != nil { + return nil, err + } + if len(info) != 1 || info[0].Key != "net/listeners/socks" { + instance.Close() + return nil, ErrCannotGetSOCKS5ProxyURL + } + proxyAddress := info[0].Val + if strings.HasPrefix(proxyAddress, "unix:") { + instance.Close() + return nil, ErrCannotGetSOCKS5ProxyURL + } + proxyURL := &url.URL{Scheme: "socks5", Host: proxyAddress} + return proxyURL, nil +} + +// maybeCleanupStateDir removes stale files inside the stateDir. +func maybeCleanupStateDir(logger model.Logger, stateDir, logFile string) { + maybeRemoveFile(logger, logFile) + maybeRemoveWithGlob(logger, filepath.Join(stateDir, "torrc-*")) + maybeRemoveWithGlob(logger, filepath.Join(stateDir, "control-port-*")) +} + +// maybeRemoveWithGlob globs and removes files. +func maybeRemoveWithGlob(logger model.Logger, pattern string) { + files, _ := filepath.Glob(pattern) + for _, file := range files { + maybeRemoveFile(logger, file) + } +} + +// maybeRemoveFile removes a file if needed. +func maybeRemoveFile(logger model.Logger, file string) { + logger.Infof("tortunnel: rm -f %s", file) + os.Remove(file) +} + +// logCommandLineArguments logs the command line arguments we're using. +func logCommandLineArguments(logger model.Logger, args []string) { + quoted := shellx.QuotedCommandLineUnsafe("tor", args...) + logger.Infof("tortunnel: command line: %s", quoted) +} diff --git a/internal/tortunnel-old/tunnel.go b/internal/tortunnel-old/tunnel.go new file mode 100644 index 0000000000..7687c1130f --- /dev/null +++ b/internal/tortunnel-old/tunnel.go @@ -0,0 +1,79 @@ +package tortunnel + +import ( + "context" + "net/url" + "sync" + "time" + + "github.com/cretz/bine/tor" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// Tunnel is a [model.Tunnel] implemented using tor. +type Tunnel struct { + // bootstrapTime is the duration of the bootstrap. + bootstrapTime time.Duration + + // instance is the running tor instance. + instance *tor.Tor + + // maybeDeleteTunnelDir is a cleanup function that deletes + // the tunnel dir if it's a temporary directory. + maybeDeleteTunnelDir func() + + // name is the tunnel name. + name string + + // proxy is the SOCKS5 proxy URL. + proxy *url.URL + + // stopOnce allows us to call stop just once. + stopOnce sync.Once +} + +var _ model.Tunnel = &Tunnel{} + +// BootstrapTime implements model.Tunnel +func (t *Tunnel) BootstrapTime() time.Duration { + return t.bootstrapTime +} + +// LookupProbeIP implements model.Tunnel +func (t *Tunnel) LookupProbeIP(ctx context.Context) (string, error) { + panic("not implemented") +} + +// Name implements model.Tunnel +func (t *Tunnel) Name() string { + return t.name +} + +// NewDNSOverHTTPSResolver implements model.Tunnel +func (t *Tunnel) NewDNSOverHTTPSResolver(logger model.Logger, URL string) model.Resolver { + httptxp := t.NewHTTPTransport(logger) + dnsxp := netxlite.NewDNSOverHTTPSTransportWithHTTPTransport(httptxp, URL) + return netxlite.WrapResolver(logger, netxlite.NewUnwrappedParallelResolver(dnsxp)) +} + +// NewHTTPTransport implements model.Tunnel +func (t *Tunnel) NewHTTPTransport(logger model.Logger) model.HTTPTransport { + dialer := netxlite.NewDialerWithoutResolver(logger) + dialer = netxlite.MaybeWrapWithProxyDialer(dialer, t.proxy) + tlsDialer := netxlite.NewTLSDialer(dialer, netxlite.NewTLSHandshakerStdlib(logger)) + return netxlite.NewHTTPTransport(logger, dialer, tlsDialer) +} + +// SOCKS5ProxyURL implements model.Tunnel +func (t *Tunnel) SOCKS5ProxyURL() *url.URL { + return t.proxy +} + +// Stop implements model.Tunnel +func (t *Tunnel) Stop() { + t.stopOnce.Do(func() { + t.instance.Close() + t.maybeDeleteTunnelDir() + }) +} diff --git a/internal/tortunnel/config.go b/internal/tortunnel/config.go new file mode 100644 index 0000000000..613692873e --- /dev/null +++ b/internal/tortunnel/config.go @@ -0,0 +1,85 @@ +package tortunnel + +// +// Config implementation +// + +import ( + "os" + + "github.com/ooni/probe-cli/v3/internal/model" +) + +// Config contains config for starting the tor tunnel. +type Config struct { + // BootstrapEvents is the OPTIONAL channel where to send + // events emitted during the bootstrap. + BootstrapEvents chan<- string `json:",omitempty"` + + // Dependencies is OPTIONAL and allow one to mock the functions + // called by Start, which is mainly useful for testing. + Dependencies *Dependencies `json:",omitempty"` + + // Logger is the OPTIONAL logger to use during the bootstrap. + Logger model.Logger `json:",omitempty"` + + // SnowflakeEnabled OPTIONALLY enables snowflake. + SnowflakeEnabled bool + + // SnowflakeRendezvousMethod is the OPTIONAL snowflake rendezvous method. + SnowflakeRendezvousMethod string + + // TunnelDir is the OPTIONAL directory in which to store state. + TunnelDir string + + // TorArgs contains OPTIONAL arguments to pass to tor. + TorArgs []string + + // TorBinary OPTIONAL tor binary path. + TorBinary string + + // TorVersion is the OPTIONAL channel where we send the version + // of tor that we are attempting to bootstrap. + TorVersion chan<- string `json:",omitempty"` +} + +// logger always returns a valid instance of [model.Logger]. +func (cc *Config) logger() model.Logger { + if cc.Logger == nil { + return model.DiscardLogger + } + return cc.Logger +} + +// dependencies always returns a valid instance of [Dependencies]. +func (cc *Config) dependencies() *Dependencies { + if cc.Dependencies == nil { + return defaultDependencies + } + return cc.Dependencies +} + +// tunnelDir returns the tunnel dir to use or an error. +func (cc *Config) tunnelDir(logger model.Logger) (string, func(), error) { + if cc.TunnelDir == "" { + dir, err := os.MkdirTemp("", "") + if err != nil { + return "", nil, err + } + return dir, cleanupTunnelDir(logger, dir), nil + } + return cc.TunnelDir, func() {}, nil +} + +// cleanupTunnelDir returns a function that removes a temporary tunnel dir. +func cleanupTunnelDir(logger model.Logger, dir string) func() { + return func() { + maybeRemoveDir(logger, dir) + } +} + +// maybeRemoveDir removes a directory if needed. +func maybeRemoveDir(logger model.Logger, dir string) { + logger.Infof("rm -rf %s", dir) + os.RemoveAll(dir) +} diff --git a/internal/tortunnel/dependencies.go b/internal/tortunnel/dependencies.go new file mode 100644 index 0000000000..21433379af --- /dev/null +++ b/internal/tortunnel/dependencies.go @@ -0,0 +1,43 @@ +package tortunnel + +// +// This package's dependencies (for testing). +// + +import ( + "context" + + "github.com/cretz/bine/control" + "github.com/cretz/bine/tor" +) + +// Dependencies contains dependencies allowing to mock [Start] for testing. +type Dependencies struct { + // Start should be equivalent to [tor.Start]. + Start func(ctx context.Context, conf *tor.StartConf) (*tor.Tor, error) + + // TorControlProtocolInfo should be equivalent to calling the + // ProtocolInfo method of instance.Control. + TorControlProtocolInfo func(instance *tor.Tor) (*control.ProtocolInfo, error) + + // TorControlGetInfo should be equivalent to calling the GetInfo + // method of the instance.Control. + TorControlGetInfo func(instance *tor.Tor, keys ...string) ([]*control.KeyVal, error) + + // TorEnableNetwork should be equivalent to calling the + // EnableNetwork method of instance. + TorEnableNetwork func(ctx context.Context, instance *tor.Tor, wait bool) error +} + +var defaultDependencies = &Dependencies{ + Start: tor.Start, + TorControlProtocolInfo: func(instance *tor.Tor) (*control.ProtocolInfo, error) { + return instance.Control.ProtocolInfo() + }, + TorControlGetInfo: func(instance *tor.Tor, keys ...string) ([]*control.KeyVal, error) { + return instance.Control.GetInfo(keys...) + }, + TorEnableNetwork: func(ctx context.Context, instance *tor.Tor, wait bool) error { + return instance.EnableNetwork(ctx, wait) + }, +} diff --git a/internal/tortunnel/doc.go b/internal/tortunnel/doc.go new file mode 100644 index 0000000000..001fc22ba5 --- /dev/null +++ b/internal/tortunnel/doc.go @@ -0,0 +1,2 @@ +// Package tortunnel allows to use tor as a [model.Tunnel]. +package tortunnel diff --git a/internal/tortunnel/tunnel.go b/internal/tortunnel/tunnel.go new file mode 100644 index 0000000000..f513429f68 --- /dev/null +++ b/internal/tortunnel/tunnel.go @@ -0,0 +1,7 @@ +package tortunnel + +// Tunnel is a [model.Tunnel] using tor. +type Tunnel struct{} + +// New creates a new instance of [Tunnel]. +func New() *Tunnel {}