diff --git a/cmd/ooniprobe/internal/cli/geoip/geoip.go b/cmd/ooniprobe/internal/cli/geoip/geoip.go index 21d4ea59d8..27ddf572cb 100644 --- a/cmd/ooniprobe/internal/cli/geoip/geoip.go +++ b/cmd/ooniprobe/internal/cli/geoip/geoip.go @@ -8,7 +8,6 @@ import ( "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/cli/root" "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/ooni" "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/output" - "github.com/ooni/probe-cli/v3/internal/model" ) func init() { @@ -37,23 +36,26 @@ func dogeoip(config dogeoipconfig) error { return err } - engine, err := probeCLI.NewProbeEngine(context.Background(), model.RunTypeManual) - if err != nil { + sess := probeCLI.NewSession(context.Background(), "manual") + defer sess.Close() + + if err := sess.Bootstrap(context.Background()); err != nil { + log.WithError(err).Error("Failed to bootstrap the measurement session") return err } - defer engine.Close() - err = engine.MaybeLookupLocation() + location, err := sess.Geolocate(context.Background()) if err != nil { + log.WithError(err).Error("Failed to lookup the location of the probe") return err } config.Logger.WithFields(log.Fields{ "type": "table", - "asn": engine.ProbeASNString(), - "network_name": engine.ProbeNetworkName(), - "country_code": engine.ProbeCC(), - "ip": engine.ProbeIP(), + "asn": location.ProbeASNString(), + "network_name": location.ProbeNetworkName(), + "country_code": location.ProbeCC(), + "ip": location.ProbeIP(), }).Info("Looked up your location") return nil diff --git a/cmd/ooniprobe/internal/nettests/dash.go b/cmd/ooniprobe/internal/nettests/dash.go index 80f8618dff..e9bcc588ce 100644 --- a/cmd/ooniprobe/internal/nettests/dash.go +++ b/cmd/ooniprobe/internal/nettests/dash.go @@ -1,14 +1,12 @@ package nettests +import "errors" + // Dash test implementation type Dash struct { } // Run starts the test func (d Dash) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder("dash") - if err != nil { - return err - } - return ctl.Run(builder, []string{""}) + return errors.New("not implemented") } diff --git a/cmd/ooniprobe/internal/nettests/dnscheck.go b/cmd/ooniprobe/internal/nettests/dnscheck.go index beebcd306d..e305dad00b 100644 --- a/cmd/ooniprobe/internal/nettests/dnscheck.go +++ b/cmd/ooniprobe/internal/nettests/dnscheck.go @@ -1,42 +1,13 @@ package nettests import ( - "context" - - engine "github.com/ooni/probe-cli/v3/internal/engine" - "github.com/ooni/probe-cli/v3/internal/model" + "errors" ) // DNSCheck nettest implementation. type DNSCheck struct{} -func (n DNSCheck) lookupURLs(ctl *Controller) ([]string, error) { - inputloader := &engine.InputLoader{ - CheckInConfig: &model.OOAPICheckInConfig{ - // not needed because we have default static input in the engine - }, - ExperimentName: "dnscheck", - InputPolicy: model.InputOrStaticDefault, - Session: ctl.Session, - SourceFiles: ctl.InputFiles, - StaticInputs: ctl.Inputs, - } - testlist, err := inputloader.Load(context.Background()) - if err != nil { - return nil, err - } - return ctl.BuildAndSetInputIdxMap(testlist) -} - // Run starts the nettest. func (n DNSCheck) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder("dnscheck") - if err != nil { - return err - } - urls, err := n.lookupURLs(ctl) - if err != nil { - return err - } - return ctl.Run(builder, urls) + return errors.New("not implemented") } diff --git a/cmd/ooniprobe/internal/nettests/facebook_messenger.go b/cmd/ooniprobe/internal/nettests/facebook_messenger.go index 1316babee5..cafaa0f361 100644 --- a/cmd/ooniprobe/internal/nettests/facebook_messenger.go +++ b/cmd/ooniprobe/internal/nettests/facebook_messenger.go @@ -1,16 +1,12 @@ package nettests +import "errors" + // FacebookMessenger test implementation type FacebookMessenger struct { } // Run starts the test func (h FacebookMessenger) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder( - "facebook_messenger", - ) - if err != nil { - return err - } - return ctl.Run(builder, []string{""}) + return errors.New("not implemented") } diff --git a/cmd/ooniprobe/internal/nettests/http_header_field_manipulation.go b/cmd/ooniprobe/internal/nettests/http_header_field_manipulation.go index 6fdd39688f..7cbe17793d 100644 --- a/cmd/ooniprobe/internal/nettests/http_header_field_manipulation.go +++ b/cmd/ooniprobe/internal/nettests/http_header_field_manipulation.go @@ -1,16 +1,12 @@ package nettests +import "errors" + // HTTPHeaderFieldManipulation test implementation type HTTPHeaderFieldManipulation struct { } // Run starts the test func (h HTTPHeaderFieldManipulation) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder( - "http_header_field_manipulation", - ) - if err != nil { - return err - } - return ctl.Run(builder, []string{""}) + return errors.New("not implemented") } diff --git a/cmd/ooniprobe/internal/nettests/http_invalid_request_line.go b/cmd/ooniprobe/internal/nettests/http_invalid_request_line.go index fb87e462d6..654a3cecc9 100644 --- a/cmd/ooniprobe/internal/nettests/http_invalid_request_line.go +++ b/cmd/ooniprobe/internal/nettests/http_invalid_request_line.go @@ -1,16 +1,12 @@ package nettests +import "errors" + // HTTPInvalidRequestLine test implementation type HTTPInvalidRequestLine struct { } // Run starts the test func (h HTTPInvalidRequestLine) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder( - "http_invalid_request_line", - ) - if err != nil { - return err - } - return ctl.Run(builder, []string{""}) + return errors.New("not implemented") } diff --git a/cmd/ooniprobe/internal/nettests/ndt.go b/cmd/ooniprobe/internal/nettests/ndt.go index b8848b0056..1368ee5c47 100644 --- a/cmd/ooniprobe/internal/nettests/ndt.go +++ b/cmd/ooniprobe/internal/nettests/ndt.go @@ -1,15 +1,12 @@ package nettests +import "errors" + // NDT test implementation. We use v7 of NDT since 2020-03-12. type NDT struct { } // Run starts the test func (n NDT) Run(ctl *Controller) error { - // Since 2020-03-18 probe-engine exports v7 as "ndt". - builder, err := ctl.Session.NewExperimentBuilder("ndt") - if err != nil { - return err - } - return ctl.Run(builder, []string{""}) + return errors.New("not implemented") } diff --git a/cmd/ooniprobe/internal/nettests/nettests.go b/cmd/ooniprobe/internal/nettests/nettests.go index bee5be39f7..3f9ea2ddb4 100644 --- a/cmd/ooniprobe/internal/nettests/nettests.go +++ b/cmd/ooniprobe/internal/nettests/nettests.go @@ -10,8 +10,8 @@ import ( "github.com/fatih/color" "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/ooni" "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/output" - engine "github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/nettests" "github.com/pkg/errors" ) @@ -22,7 +22,8 @@ type Nettest interface { // NewController creates a nettest controller func NewController( - nt Nettest, probe *ooni.Probe, res *model.DatabaseResult, sess *engine.Session) *Controller { + nt Nettest, probe *ooni.Probe, res *model.DatabaseResult, + sess *nettests.Session) *Controller { return &Controller{ Probe: probe, nt: nt, @@ -34,15 +35,16 @@ func NewController( // Controller is passed to the run method of every Nettest // each nettest instance has one controller type Controller struct { - Probe *ooni.Probe - Session *engine.Session - res *model.DatabaseResult - nt Nettest - ntCount int - ntIndex int - ntStartTime time.Time // used to calculate the eta - msmts map[int64]*model.DatabaseMeasurement - inputIdxMap map[int64]int64 // Used to map mk idx to database id + CheckInResult *model.OOAPICheckInResult + Probe *ooni.Probe + Session *nettests.Session + res *model.DatabaseResult + nt Nettest + ntCount int + ntIndex int + ntStartTime time.Time // used to calculate the eta + msmts map[int64]*model.DatabaseMeasurement + inputIdxMap map[int64]int64 // Used to map mk idx to database id // InputFiles optionally contains the names of the input // files to read inputs from (only for nettests that take @@ -120,13 +122,12 @@ func (c *Controller) SetNettestIndex(i, n int) { // // This function will continue to run in most cases but will // immediately halt if something's wrong with the file system. -func (c *Controller) Run(builder model.ExperimentBuilder, inputs []string) error { +func (c *Controller) Run(factory nettests.ExperimentFactory, inputs []string) error { db := c.Probe.DB() + c.numInputs = len(inputs) // This will configure the controller as handler for the callbacks // called by ooni/probe-engine/experiment.Experiment. - builder.SetCallbacks(model.ExperimentCallbacks(c)) - c.numInputs = len(inputs) - exp := builder.NewExperiment() + exp := factory.NewExperiment(c) defer func() { c.res.DataUsageDown += exp.KibiBytesReceived() c.res.DataUsageUp += exp.KibiBytesSent() @@ -142,14 +143,7 @@ func (c *Controller) Run(builder model.ExperimentBuilder, inputs []string) error log.Debug(color.RedString("status.started")) if c.Probe.Config().Sharing.UploadResults { - if err := exp.OpenReportContext(context.Background()); err != nil { - log.Debugf( - "%s: %s", color.RedString("failure.report_create"), err.Error(), - ) - } else { - log.Debugf(color.RedString("status.report_create")) - reportID = sql.NullString{String: exp.ReportID(), Valid: true} - } + reportID = sql.NullString{String: exp.ReportID(), Valid: true} } maxRuntime := time.Duration(c.Probe.Config().Nettests.WebsitesMaxRuntime) * time.Second @@ -196,7 +190,7 @@ func (c *Controller) Run(builder model.ExperimentBuilder, inputs []string) error if input != "" { c.OnProgress(0, fmt.Sprintf("processing input: %s", input)) } - measurement, err := exp.MeasureWithContext(context.Background(), input) + measurement, err := exp.Measure(context.Background(), input) if err != nil { log.WithError(err).Debug(color.RedString("failure.measurement")) if err := db.Failed(c.msmts[idx64], err.Error()); err != nil { @@ -217,7 +211,7 @@ func (c *Controller) Run(builder model.ExperimentBuilder, inputs []string) error // Implementation note: SubmitMeasurement will fail here if we did fail // to open the report but we still want to continue. There will be a // bit of a spew in the logs, perhaps, but stopping seems less efficient. - if err := exp.SubmitAndUpdateMeasurementContext(context.Background(), measurement); err != nil { + if err := c.Session.Submit(context.Background(), measurement); err != nil { log.Debug(color.RedString("failure.measurement_submission")) if err := db.UploadFailed(c.msmts[idx64], err.Error()); err != nil { return errors.Wrap(err, "failed to mark upload as failed") @@ -231,7 +225,7 @@ func (c *Controller) Run(builder model.ExperimentBuilder, inputs []string) error } // We only save the measurement to disk if we failed to upload the measurement if saveToDisk { - if err := exp.SaveMeasurement(measurement, msmt.MeasurementFilePath.String); err != nil { + if err := nettests.SaveMeasurement(measurement, msmt.MeasurementFilePath.String); err != nil { return errors.Wrap(err, "failed to save measurement on disk") } } diff --git a/cmd/ooniprobe/internal/nettests/nettests_test.go b/cmd/ooniprobe/internal/nettests/nettests_test.go index 0b4d94d703..83ae8e96d3 100644 --- a/cmd/ooniprobe/internal/nettests/nettests_test.go +++ b/cmd/ooniprobe/internal/nettests/nettests_test.go @@ -48,7 +48,7 @@ func TestRun(t *testing.T) { t.Skip("skip test in short mode") } probe := newOONIProbe(t) - sess, err := probe.NewSession(context.Background(), model.RunTypeManual) + sess, err := probe.NewProbeEngine(context.Background(), model.RunTypeManual) if err != nil { t.Fatal(err) } diff --git a/cmd/ooniprobe/internal/nettests/psiphon.go b/cmd/ooniprobe/internal/nettests/psiphon.go index 940340cc4e..44dc51ef52 100644 --- a/cmd/ooniprobe/internal/nettests/psiphon.go +++ b/cmd/ooniprobe/internal/nettests/psiphon.go @@ -1,16 +1,12 @@ package nettests +import "errors" + // Psiphon test implementation type Psiphon struct { } // Run starts the test func (h Psiphon) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder( - "psiphon", - ) - if err != nil { - return err - } - return ctl.Run(builder, []string{""}) + return errors.New("not implemented") } diff --git a/cmd/ooniprobe/internal/nettests/riseupvpn.go b/cmd/ooniprobe/internal/nettests/riseupvpn.go index 185fbcefe4..ccf041f856 100644 --- a/cmd/ooniprobe/internal/nettests/riseupvpn.go +++ b/cmd/ooniprobe/internal/nettests/riseupvpn.go @@ -1,17 +1,12 @@ package nettests +import "errors" + // RiseupVPN test implementation type RiseupVPN struct { } // Run starts the test func (h RiseupVPN) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder( - "riseupvpn", - ) - if err != nil { - return err - } - - return ctl.Run(builder, []string{""}) + return errors.New("not implemented") } diff --git a/cmd/ooniprobe/internal/nettests/run.go b/cmd/ooniprobe/internal/nettests/run.go index 25385a647e..59347fa439 100644 --- a/cmd/ooniprobe/internal/nettests/run.go +++ b/cmd/ooniprobe/internal/nettests/run.go @@ -9,6 +9,7 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/ooni" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/platform" "github.com/pkg/errors" ) @@ -59,26 +60,52 @@ func RunGroup(config RunGroupConfig) error { return nil } - sess, err := config.Probe.NewSession(context.Background(), config.RunType) - if err != nil { - log.WithError(err).Error("Failed to create a measurement session") + sess := config.Probe.NewSession(context.Background(), config.RunType) + defer sess.Close() + + if err := sess.Bootstrap(context.Background()); err != nil { + log.WithError(err).Error("Failed to bootstrap the measurement session") return err } - defer sess.Close() - err = sess.MaybeLookupLocation() + location, err := sess.Geolocate(context.Background()) if err != nil { log.WithError(err).Error("Failed to lookup the location of the probe") return err } db := config.Probe.DB() - network, err := db.CreateNetwork(sess) + network, err := db.CreateNetwork(location) if err != nil { log.WithError(err).Error("Failed to create the network row") return err } - if err := sess.MaybeLookupBackends(); err != nil { - log.WithError(err).Warn("Failed to discover OONI backends") + + log.Debugf( + "Enabled category codes are the following %v", + config.Probe.Config().Nettests.WebsitesEnabledCategoryCodes, + ) + checkInConfig := &model.OOAPICheckInConfig{ + // Setting Charging and OnWiFi to true causes the CheckIn + // API to return to us as much URL as possible with the + // given RunType hint. + Charging: true, + OnWiFi: true, + Platform: platform.Name(), + ProbeASN: location.ProbeASNString(), + ProbeCC: location.ProbeCC(), + RunType: config.RunType, + SoftwareName: sess.BootstrapRequest().SoftwareName, + SoftwareVersion: sess.BootstrapRequest().SoftwareVersion, + WebConnectivity: model.OOAPICheckInConfigWebConnectivity{ + CategoryCodes: config.Probe.Config().Nettests.WebsitesEnabledCategoryCodes, + }, + } + if checkInConfig.WebConnectivity.CategoryCodes == nil { + checkInConfig.WebConnectivity.CategoryCodes = []string{} + } + checkInResult, err := sess.CheckIn(context.Background(), checkInConfig) + if err != nil { + log.WithError(err).Warn("Failed to query the check-in API") return err } @@ -111,6 +138,7 @@ func RunGroup(config RunGroupConfig) error { } log.Debugf("Running test %T", nt) ctl := NewController(nt, config.Probe, result, sess) + ctl.CheckInResult = checkInResult ctl.InputFiles = config.InputFiles ctl.Inputs = config.Inputs ctl.RunType = config.RunType diff --git a/cmd/ooniprobe/internal/nettests/signal.go b/cmd/ooniprobe/internal/nettests/signal.go index 3a2df6b874..481e6df09f 100644 --- a/cmd/ooniprobe/internal/nettests/signal.go +++ b/cmd/ooniprobe/internal/nettests/signal.go @@ -1,15 +1,11 @@ package nettests +import "errors" + // Signal nettest implementation. type Signal struct{} // Run starts the nettest. func (h Signal) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder( - "signal", - ) - if err != nil { - return err - } - return ctl.Run(builder, []string{""}) + return errors.New("not implemented") } diff --git a/cmd/ooniprobe/internal/nettests/stunreachability.go b/cmd/ooniprobe/internal/nettests/stunreachability.go index 186bb7bb98..f8fa58377b 100644 --- a/cmd/ooniprobe/internal/nettests/stunreachability.go +++ b/cmd/ooniprobe/internal/nettests/stunreachability.go @@ -1,42 +1,13 @@ package nettests import ( - "context" - - engine "github.com/ooni/probe-cli/v3/internal/engine" - "github.com/ooni/probe-cli/v3/internal/model" + "errors" ) // STUNReachability nettest implementation. type STUNReachability struct{} -func (n STUNReachability) lookupURLs(ctl *Controller) ([]string, error) { - inputloader := &engine.InputLoader{ - CheckInConfig: &model.OOAPICheckInConfig{ - // not needed because we have default static input in the engine - }, - ExperimentName: "stunreachability", - InputPolicy: model.InputOrStaticDefault, - Session: ctl.Session, - SourceFiles: ctl.InputFiles, - StaticInputs: ctl.Inputs, - } - testlist, err := inputloader.Load(context.Background()) - if err != nil { - return nil, err - } - return ctl.BuildAndSetInputIdxMap(testlist) -} - // Run starts the nettest. func (n STUNReachability) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder("stunreachability") - if err != nil { - return err - } - urls, err := n.lookupURLs(ctl) - if err != nil { - return err - } - return ctl.Run(builder, urls) + return errors.New("not implemented") } diff --git a/cmd/ooniprobe/internal/nettests/telegram.go b/cmd/ooniprobe/internal/nettests/telegram.go index 82d75d82c2..3f4ccd48c5 100644 --- a/cmd/ooniprobe/internal/nettests/telegram.go +++ b/cmd/ooniprobe/internal/nettests/telegram.go @@ -1,16 +1,12 @@ package nettests +import "errors" + // Telegram test implementation type Telegram struct { } // Run starts the test func (h Telegram) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder( - "telegram", - ) - if err != nil { - return err - } - return ctl.Run(builder, []string{""}) + return errors.New("not implemented") } diff --git a/cmd/ooniprobe/internal/nettests/tor.go b/cmd/ooniprobe/internal/nettests/tor.go index 96bfbb7d2f..5949136815 100644 --- a/cmd/ooniprobe/internal/nettests/tor.go +++ b/cmd/ooniprobe/internal/nettests/tor.go @@ -1,16 +1,12 @@ package nettests +import "errors" + // Tor test implementation type Tor struct { } // Run starts the test func (h Tor) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder( - "tor", - ) - if err != nil { - return err - } - return ctl.Run(builder, []string{""}) + return errors.New("not implemented") } diff --git a/cmd/ooniprobe/internal/nettests/torsf.go b/cmd/ooniprobe/internal/nettests/torsf.go index 494dae4743..8072e93ada 100644 --- a/cmd/ooniprobe/internal/nettests/torsf.go +++ b/cmd/ooniprobe/internal/nettests/torsf.go @@ -1,16 +1,14 @@ package nettests +import "errors" + // TorSf test implementation type TorSf struct { } // Run starts the test func (h TorSf) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder("torsf") - if err != nil { - return err - } - return ctl.Run(builder, []string{""}) + return errors.New("not implemented") } func (h TorSf) onlyBackground() {} diff --git a/cmd/ooniprobe/internal/nettests/vanillator.go b/cmd/ooniprobe/internal/nettests/vanillator.go index 11d7266549..e6b339dea1 100644 --- a/cmd/ooniprobe/internal/nettests/vanillator.go +++ b/cmd/ooniprobe/internal/nettests/vanillator.go @@ -1,16 +1,14 @@ package nettests +import "errors" + // VanillaTor test implementation type VanillaTor struct { } // Run starts the test func (h VanillaTor) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder("vanilla_tor") - if err != nil { - return err - } - return ctl.Run(builder, []string{""}) + return errors.New("not implemented") } func (h VanillaTor) onlyBackground() {} diff --git a/cmd/ooniprobe/internal/nettests/web_connectivity.go b/cmd/ooniprobe/internal/nettests/web_connectivity.go index fc643c0091..efbbfc6a66 100644 --- a/cmd/ooniprobe/internal/nettests/web_connectivity.go +++ b/cmd/ooniprobe/internal/nettests/web_connectivity.go @@ -1,33 +1,14 @@ package nettests import ( - "context" - - "github.com/apex/log" - engine "github.com/ooni/probe-cli/v3/internal/engine" - "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/nettests" ) -func (n WebConnectivity) lookupURLs(ctl *Controller, categories []string) ([]string, error) { - inputloader := &engine.InputLoader{ - CheckInConfig: &model.OOAPICheckInConfig{ - // Setting Charging and OnWiFi to true causes the CheckIn - // API to return to us as much URL as possible with the - // given RunType hint. - Charging: true, - OnWiFi: true, - RunType: ctl.RunType, - WebConnectivity: model.OOAPICheckInConfigWebConnectivity{ - CategoryCodes: categories, - }, - }, - ExperimentName: "web_connectivity", - InputPolicy: model.InputOrQueryBackend, - Session: ctl.Session, - SourceFiles: ctl.InputFiles, - StaticInputs: ctl.Inputs, - } - testlist, err := inputloader.Load(context.Background()) +func (n WebConnectivity) lookupURLs( + ctl *Controller, + factory *nettests.WebConnectivityFactory, +) ([]string, error) { + testlist, err := factory.LoadInputs() if err != nil { return nil, err } @@ -39,14 +20,19 @@ type WebConnectivity struct{} // Run starts the test func (n WebConnectivity) Run(ctl *Controller) error { - log.Debugf("Enabled category codes are the following %v", ctl.Probe.Config().Nettests.WebsitesEnabledCategoryCodes) - urls, err := n.lookupURLs(ctl, ctl.Probe.Config().Nettests.WebsitesEnabledCategoryCodes) + factoryConfig := &nettests.WebConnectivityFactoryConfig{ + CheckIn: ctl.CheckInResult, + InputFiles: ctl.InputFiles, + Inputs: ctl.Inputs, + Session: ctl.Session, + } + factory, err := nettests.NewWebConnectivityFactory(factoryConfig) if err != nil { return err } - builder, err := ctl.Session.NewExperimentBuilder("web_connectivity") + urls, err := n.lookupURLs(ctl, factory) if err != nil { return err } - return ctl.Run(builder, urls) + return ctl.Run(factory, urls) } diff --git a/cmd/ooniprobe/internal/nettests/whatsapp.go b/cmd/ooniprobe/internal/nettests/whatsapp.go index 4660abe006..bc8e3523eb 100644 --- a/cmd/ooniprobe/internal/nettests/whatsapp.go +++ b/cmd/ooniprobe/internal/nettests/whatsapp.go @@ -1,16 +1,12 @@ package nettests +import "errors" + // WhatsApp test implementation type WhatsApp struct { } // Run starts the test func (h WhatsApp) Run(ctl *Controller) error { - builder, err := ctl.Session.NewExperimentBuilder( - "whatsapp", - ) - if err != nil { - return err - } - return ctl.Run(builder, []string{""}) + return errors.New("not implemented") } diff --git a/cmd/ooniprobe/internal/ooni/ooni.go b/cmd/ooniprobe/internal/ooni/ooni.go index dcffdab6c4..9e4b1e1e8a 100644 --- a/cmd/ooniprobe/internal/ooni/ooni.go +++ b/cmd/ooniprobe/internal/ooni/ooni.go @@ -15,9 +15,10 @@ import ( "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/utils" "github.com/ooni/probe-cli/v3/internal/database" "github.com/ooni/probe-cli/v3/internal/engine" - "github.com/ooni/probe-cli/v3/internal/kvstore" "github.com/ooni/probe-cli/v3/internal/legacy/assetsdir" "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/nettests" + "github.com/ooni/probe-cli/v3/internal/session" "github.com/pkg/errors" ) @@ -36,17 +37,19 @@ type ProbeCLI interface { IsBatch() bool Home() string TempDir() string - NewProbeEngine(ctx context.Context, runType model.RunType) (ProbeEngine, error) + NewSession(ctx context.Context, runType model.RunType) *nettests.Session } // ProbeEngine is an instance of the OONI Probe engine. type ProbeEngine interface { + model.LocationProvider + engine.InputLoaderSession Close() error + MaybeLookupBackends(config *model.OOAPICheckInConfig) error MaybeLookupLocation() error - ProbeASNString() string - ProbeCC() string - ProbeIP() string - ProbeNetworkName() string + NewExperimentBuilder(name string) (model.ExperimentBuilder, error) + SoftwareName() string + SoftwareVersion() string } // Probe contains the ooniprobe CLI context. @@ -211,19 +214,9 @@ func (p *Probe) Init(softwareName, softwareVersion, proxy string) error { return nil } -// NewSession creates a new ooni/probe-engine session using the -// current configuration inside the context. The caller must close -// the session when done using it, by calling sess.Close(). -func (p *Probe) NewSession(ctx context.Context, runType model.RunType) (*engine.Session, error) { - kvstore, err := kvstore.NewFS( - utils.EngineDir(p.home), - ) - if err != nil { - return nil, errors.Wrap(err, "creating engine's kvstore") - } - if err := os.MkdirAll(p.tunnelDir, 0700); err != nil { - return nil, errors.Wrap(err, "creating tunnel dir") - } +// NewSession creates a new measurement session. +func (p *Probe) NewSession(ctx context.Context, runType model.RunType) *nettests.Session { + // When the software name is the default software name and we're running // in unattended mode, adjust the software name accordingly. // @@ -232,24 +225,33 @@ func (p *Probe) NewSession(ctx context.Context, runType model.RunType) (*engine. if runType == model.RunTypeTimed && softwareName == DefaultSoftwareName { softwareName = DefaultSoftwareName + "-unattended" } - return engine.NewSession(ctx, engine.SessionConfig{ - KVStore: kvstore, - Logger: logger, - SoftwareName: softwareName, - SoftwareVersion: p.softwareVersion, - TempDir: p.tempDir, - TunnelDir: p.tunnelDir, - ProxyURL: p.proxyURL, - }) -} -// NewProbeEngine creates a new ProbeEngine instance. -func (p *Probe) NewProbeEngine(ctx context.Context, runType model.RunType) (ProbeEngine, error) { - sess, err := p.NewSession(ctx, runType) - if err != nil { - return nil, err + // TODO(bassosimone): unclear to me how to set these fields + // + // - SnowflakeRendezvousMethod + // - TorArgs + // - TorBinary + // - VerboseLogging + + var proxyURL string + if p.proxyURL != nil { + proxyURL = p.proxyURL.String() } - return sess, nil + + config := &session.BootstrapRequest{ + SnowflakeRendezvousMethod: "", + StateDir: utils.EngineDir(p.home), + ProxyURL: proxyURL, + SoftwareName: softwareName, + SoftwareVersion: p.softwareVersion, + TorArgs: nil, + TorBinary: "", + TempDir: p.tempDir, + TunnelDir: p.tunnelDir, + VerboseLogging: false, + } + + return nettests.NewSession(config, logger) } // NewProbe creates a new probe instance. diff --git a/cmd/ooniprobe/internal/ooni/sessionapi.go b/cmd/ooniprobe/internal/ooni/sessionapi.go new file mode 100644 index 0000000000..3457b34c49 --- /dev/null +++ b/cmd/ooniprobe/internal/ooni/sessionapi.go @@ -0,0 +1,332 @@ +package ooni + +import ( + "context" + "encoding/json" + "errors" + "os" + "sync" + "time" + + "github.com/ooni/probe-cli/v3/internal/bytecounter" + "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/registry" + "github.com/ooni/probe-cli/v3/internal/session" +) + +// engineSession emulates [engine.engineSession] using [session.engineSession]. +type engineSession struct { + // checkIn stores the most recent check-in API response (if any). + checkIn model.OptionalPtr[model.OOAPICheckInResult] + + // checkInMu protects the checkIn field + checkInMu sync.Mutex + + // config contains the config for bootstrapping the session. + config *session.BootstrapRequest + + // logger is the logger to use. + logger model.Logger + + // once allows us to run cleanups just once. + once sync.Once + + // session is the initially empty session. + session *session.Session +} + +var ( + _ ProbeEngine = &engineSession{} + _ engine.InputLoaderSession = &engineSession{} + _ model.LocationProvider = &engineSession{} +) + +// newSession creates a new instance of Session. +func newSession(config *session.BootstrapRequest, logger model.Logger) *engineSession { + return &engineSession{ + checkIn: model.OptionalPtr[model.OOAPICheckInResult]{}, + checkInMu: sync.Mutex{}, + config: config, + logger: logger, + once: sync.Once{}, + session: session.New(), + } +} + +// SoftwareName implements ProbeEngine +func (s *engineSession) SoftwareName() string { + return s.config.SoftwareName +} + +// SoftwareVersion implements ProbeEngine +func (s *engineSession) SoftwareVersion() string { + return s.config.SoftwareVersion +} + +// Close implements ProbeEngine +func (s *engineSession) Close() error { + s.once.Do(func() { + s.session.Close() + }) + return nil +} + +// MaybeLookupLocation implements ProbeEngine +func (s *engineSession) MaybeLookupLocation() error { + if _, err := s.maybeLookupLocation(); err != nil { + return err + } + return nil +} + +// ProbeASNString implements ProbeEngine +func (s *engineSession) ProbeASNString() string { + location, err := s.maybeLookupLocation() + if err != nil { + return model.DefaultProbeASNString + } + return location.ProbeASNString() +} + +// ProbeCC implements ProbeEngine +func (s *engineSession) ProbeCC() string { + location, err := s.maybeLookupLocation() + if err != nil { + return model.DefaultProbeCC + } + return location.ProbeCC() +} + +// ProbeIP implements ProbeEngine +func (s *engineSession) ProbeIP() string { + location, err := s.maybeLookupLocation() + if err != nil { + return model.DefaultProbeIP + } + return location.ProbeIP() +} + +// ProbeNetworkName implements ProbeEngine +func (s *engineSession) ProbeNetworkName() string { + location, err := s.maybeLookupLocation() + if err != nil { + return model.DefaultProbeNetworkName + } + return location.ProbeNetworkName() +} + +// ProbeASN implements model.LocationProvider +func (s *engineSession) ProbeASN() uint { + location, err := s.maybeLookupLocation() + if err != nil { + return model.DefaultProbeASN + } + return location.ProbeASN() +} + +// ResolverASN implements model.LocationProvider +func (s *engineSession) ResolverASN() uint { + location, err := s.maybeLookupLocation() + if err != nil { + return model.DefaultResolverASN + } + return location.ResolverASN() +} + +// ResolverASNString implements model.LocationProvider +func (s *engineSession) ResolverASNString() string { + location, err := s.maybeLookupLocation() + if err != nil { + return model.DefaultResolverASNString + } + return location.ResolverASNString() +} + +// ResolverIP implements model.LocationProvider +func (s *engineSession) ResolverIP() string { + location, err := s.maybeLookupLocation() + if err != nil { + return model.DefaultResolverIP + } + return location.ResolverIP() +} + +// ResolverNetworkName implements model.LocationProvider +func (s *engineSession) ResolverNetworkName() string { + location, err := s.maybeLookupLocation() + if err != nil { + return model.DefaultResolverNetworkName + } + return location.ResolverNetworkName() +} + +// MaybeLookupBackends implements ProbeEngine +func (s *engineSession) MaybeLookupBackends(config *model.OOAPICheckInConfig) error { + _, err := s.maybeCheckIn(context.Background(), config) + return err +} + +// CheckIn implements engine.InputLoaderSession +func (s *engineSession) CheckIn( + ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResultNettests, error) { + return s.maybeCheckIn(ctx, config) +} + +// NewExperimentBuilder implements ProbeEngine +func (s *engineSession) NewExperimentBuilder(name string) (model.ExperimentBuilder, error) { + factory, err := registry.NewFactory(name) + if err != nil { + return nil, err + } + + // Lock because we are accessing the cached check-in + defer s.checkInMu.Unlock() + s.checkInMu.Lock() + if s.checkIn.IsNone() { + return nil, errors.New("no cached check-in API response") + } + resp := s.checkIn.Unwrap() + + var reportID string + switch name { + case "web_connectivity": + if resp.Tests.WebConnectivity == nil { + return nil, errors.New("no experiment-specific info in check-in API response") + } + reportID = resp.Tests.WebConnectivity.ReportID + default: + return nil, errors.New("not implemented") + } + + meb := &modelExperimentBuilder{ + callbacks: nil, + factory: factory, + reportID: reportID, + session: s, + } + return meb, nil +} + +// modelExperimentBuilder implements [model.ExperimentBuilder] using [session.Session]. +type modelExperimentBuilder struct { + callbacks model.ExperimentCallbacks + factory *registry.Factory + reportID string + session *engineSession +} + +var _ model.ExperimentBuilder = &modelExperimentBuilder{} + +// InputPolicy implements model.ExperimentBuilder +func (meb *modelExperimentBuilder) InputPolicy() model.InputPolicy { + return meb.factory.InputPolicy() +} + +// Interruptible implements model.ExperimentBuilder +func (meb *modelExperimentBuilder) Interruptible() bool { + return meb.factory.Interruptible() +} + +// NewExperiment implements model.ExperimentBuilder +func (meb *modelExperimentBuilder) NewExperiment() model.Experiment { + measurer := meb.factory.NewExperimentMeasurer() + me := &modelExperiment{ + bc: bytecounter.New(), + measurer: measurer, + meb: meb, + testStartTime: time.Now(), + } + return me +} + +// Options implements model.ExperimentBuilder +func (meb *modelExperimentBuilder) Options() (map[string]model.ExperimentOptionInfo, error) { + return meb.factory.Options() +} + +// SetCallbacks implements model.ExperimentBuilder +func (meb *modelExperimentBuilder) SetCallbacks(callbacks model.ExperimentCallbacks) { + meb.callbacks = callbacks +} + +// SetOptionAny implements model.ExperimentBuilder +func (meb *modelExperimentBuilder) SetOptionAny(key string, value any) error { + return meb.factory.SetOptionAny(key, value) +} + +// SetOptionsAny implements model.ExperimentBuilder +func (meb *modelExperimentBuilder) SetOptionsAny(options map[string]any) error { + return meb.factory.SetOptionsAny(options) +} + +// modelExperiment implements [model.Experiment] using [session.Session]. +type modelExperiment struct { + bc *bytecounter.Counter + measurer model.ExperimentMeasurer + meb *modelExperimentBuilder + testStartTime time.Time +} + +var _ model.Experiment = &modelExperiment{} + +// GetSummaryKeys implements model.Experiment +func (me *modelExperiment) GetSummaryKeys(m *model.Measurement) (any, error) { + return me.measurer.GetSummaryKeys(m) +} + +// KibiBytesReceived implements model.Experiment +func (me *modelExperiment) KibiBytesReceived() float64 { + return me.bc.KibiBytesReceived() +} + +// KibiBytesSent implements model.Experiment +func (me *modelExperiment) KibiBytesSent() float64 { + return me.bc.KibiBytesSent() +} + +// MeasureAsync implements model.Experiment +func (me *modelExperiment) MeasureAsync(ctx context.Context, input string) (<-chan *model.Measurement, error) { + return nil, errors.New("not implemented") +} + +// MeasureWithContext implements model.Experiment +func (me *modelExperiment) MeasureWithContext(ctx context.Context, input string) (*model.Measurement, error) { + // XXX: bytes sent and received? + switch me.measurer.ExperimentName() { + case "web_connectivity": + return me.runWebConnectivity(ctx, input) + default: + return nil, errors.New("not implemented") + } +} + +// Name implements model.Experiment +func (me *modelExperiment) Name() string { + return me.measurer.ExperimentName() +} + +// OpenReportContext implements model.Experiment +func (me *modelExperiment) OpenReportContext(ctx context.Context) error { + return nil +} + +// ReportID implements model.Experiment +func (me *modelExperiment) ReportID() string { + return me.meb.reportID +} + +// SaveMeasurement implements model.Experiment +func (me *modelExperiment) SaveMeasurement(measurement *model.Measurement, filePath string) error { + data, err := json.Marshal(measurement) + if err != nil { + return err + } + return os.WriteFile(filePath, data, 0600) +} + +// SubmitAndUpdateMeasurementContext implements model.Experiment +func (me *modelExperiment) SubmitAndUpdateMeasurementContext(ctx context.Context, measurement *model.Measurement) error { + // Note: the measurement has already the correct reportID since the beginning + return me.submit(ctx, measurement) +} diff --git a/cmd/ooniprobe/internal/ooni/sessionimpl.go b/cmd/ooniprobe/internal/ooni/sessionimpl.go new file mode 100644 index 0000000000..a3d6825a85 --- /dev/null +++ b/cmd/ooniprobe/internal/ooni/sessionimpl.go @@ -0,0 +1,174 @@ +package ooni + +import ( + "context" + + "github.com/ooni/probe-cli/v3/internal/geolocate" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/session" +) + +// maybeBootstrap performs the session's bootstrap. +func (s *engineSession) maybeBootstrap() error { + ctx := context.Background() // XXX + if err := s.session.Send(ctx, &session.Request{Bootstrap: s.config}); err != nil { + return err + } + for { + resp, err := s.session.Recv(ctx) + if err != nil { + return err + } + if resp.Log != nil { + s.emitLog(resp.Log) + continue + } + if resp.Ticker != nil { + s.logger.Infof("bootstrap in progress (elapsed: %+v)", resp.Ticker.ElapsedTime) + continue + } + if resp.Bootstrap != nil { + return resp.Bootstrap.Error + } + s.logger.Warnf("unexpected event: %+v", resp) + } +} + +// maybeLookupLocation geolocates the probe. +func (s *engineSession) maybeLookupLocation() (*geolocate.Results, error) { + ctx := context.Background() // XXX + if err := s.maybeBootstrap(); err != nil { + return nil, err + } + req := &session.GeolocateRequest{} + if err := s.session.Send(ctx, &session.Request{Geolocate: req}); err != nil { + return nil, err + } + for { + resp, err := s.session.Recv(ctx) + if err != nil { + return nil, err + } + if resp.Log != nil { + s.emitLog(resp.Log) + continue + } + if resp.Ticker != nil { + s.logger.Infof("geolocate in progress (elapsed: %+v)", resp.Ticker.ElapsedTime) + continue + } + if resp.Geolocate != nil { + return resp.Geolocate.Location, resp.Geolocate.Error + } + s.logger.Warnf("unexpected event: %+v", resp) + } +} + +// maybeCheckIn calls the check-in API. +func (s *engineSession) maybeCheckIn( + ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResultNettests, error) { + if err := s.maybeBootstrap(); err != nil { + return nil, err + } + if err := s.session.Send(ctx, &session.Request{CheckIn: config}); err != nil { + return nil, err + } + for { + resp, err := s.session.Recv(ctx) + if err != nil { + return nil, err + } + if resp.Log != nil { + s.emitLog(resp.Log) + continue + } + if resp.Ticker != nil { + s.logger.Infof("check-in in progress (elapsed: %+v)", resp.Ticker.ElapsedTime) + continue + } + if resp.CheckIn != nil { + if resp.CheckIn.Error != nil { + return nil, resp.CheckIn.Error + } + // While this code has been writting with single-goroutine usage + // in mind, it seems safer to protect this variable anyway + s.checkInMu.Lock() + s.checkIn = model.NewOptionalPtr(resp.CheckIn.Result) + s.checkInMu.Unlock() + return &resp.CheckIn.Result.Tests, nil + } + s.logger.Warnf("unexpected event: %+v", resp) + } +} + +// emitLog emits a log event. +func (s *engineSession) emitLog(ev *session.LogEvent) { + switch ev.Level { + case "DEBUG": + s.logger.Debug(ev.Message) + case "WARNING": + s.logger.Warn(ev.Message) + default: + s.logger.Info(ev.Message) + } +} + +// runWebConnectivity runs the Web Connectivity experiment. +func (me *modelExperiment) runWebConnectivity( + ctx context.Context, input string) (*model.Measurement, error) { + runtimex.Assert(me.measurer.ExperimentName() == "web_connectivity", "invalid experiment") + req := &session.WebConnectivityRequest{ + Input: input, + ReportID: me.meb.reportID, + TestStartTime: me.testStartTime, + } + sess := me.meb.session.session + if err := sess.Send(ctx, &session.Request{WebConnectivity: req}); err != nil { + return nil, err + } + for { + resp, err := sess.Recv(ctx) + if err != nil { + return nil, err + } + if resp.Log != nil { + me.meb.session.emitLog(resp.Log) + continue + } + if resp.Ticker != nil { + me.meb.session.logger.Infof("check-in in progress (elapsed: %+v)", resp.Ticker.ElapsedTime) + continue + } + if resp.WebConnectivity != nil { + return resp.WebConnectivity.Measurement, resp.WebConnectivity.Error + } + me.meb.session.logger.Warnf("unexpected event: %+v", resp) + } +} + +// submit submites a measurement. +func (me *modelExperiment) submit(ctx context.Context, measurement *model.Measurement) error { + sess := me.meb.session.session + if err := sess.Send(ctx, &session.Request{Submit: measurement}); err != nil { + return err + } + for { + resp, err := sess.Recv(ctx) + if err != nil { + return err + } + if resp.Log != nil { + me.meb.session.emitLog(resp.Log) + continue + } + if resp.Ticker != nil { + me.meb.session.logger.Infof("webconnectivity (elapsed: %+v)", resp.Ticker.ElapsedTime) + continue + } + if resp.Submit != nil { + return resp.Submit.Error + } + me.meb.session.logger.Warnf("unexpected event: %+v", resp) + } +} diff --git a/internal/backendclient/backendclient.go b/internal/backendclient/backendclient.go new file mode 100644 index 0000000000..7b46fc85f7 --- /dev/null +++ b/internal/backendclient/backendclient.go @@ -0,0 +1,92 @@ +// Package backendclient implements a client to communicate +// with the OONI backend infrastructure. +package backendclient + +import ( + "context" + "errors" + "net/url" + + "github.com/ooni/probe-cli/v3/internal/httpapi" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/ooapi" +) + +// Config contains configuration for [New]. +type Config struct { + // BaseURL is the OPTIONAL OONI backend URL. + BaseURL *url.URL + + // KVStore is the MANDATORY key-value store to use. + KVStore model.KeyValueStore + + // HTTPClient is the MANDATORY underlying HTTPClient to use. + HTTPClient model.HTTPClient + + // Logger is the MANDATORY logger to use. + Logger model.Logger + + // UserAgent is the MANDATORY user agent to use. + UserAgent string +} + +// Client is a client to communicate with the OONI backend. +type Client struct { + // endpoint is the HTTP API endpoint. + endpoint *httpapi.Endpoint +} + +// New constructs a new instance of [Client]. +func New(config *Config) *Client { + baseURL := "https://api.ooni.io/" + if config.BaseURL != nil { + baseURL = config.BaseURL.String() + } + endpoint := &httpapi.Endpoint{ + BaseURL: baseURL, + HTTPClient: config.HTTPClient, + Host: "", // no need to configure + Logger: config.Logger, + UserAgent: config.UserAgent, + } + backendClient := &Client{ + endpoint: endpoint, + } + return backendClient +} + +// CheckIn invokes the check-in API. +func (c *Client) CheckIn( + ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResult, error) { + op := measurexlite.NewOperationLogger( + c.endpoint.Logger, + "backendclient: check-in using %s", + c.endpoint.BaseURL, + ) + r, err := httpapi.Call(ctx, ooapi.NewDescriptorCheckIn(config), c.endpoint) + op.Stop(err) + return r, err +} + +// FetchPsiphonConfig retrieves Psiphon configuration. +func (c *Client) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { + return nil, errors.New("not implemented") +} + +// FetchTorTargets fetches measurement targets for the tor experiment. +func (c *Client) FetchTorTargets( + ctx context.Context, cc string) (result map[string]model.OOAPITorTarget, err error) { + return nil, errors.New("not implemented") +} + +// Submit submits the given measurement. +func (c *Client) Submit(ctx context.Context, m *model.Measurement) error { + req := &model.OOAPICollectorUpdateRequest{ + Format: "json", + Content: m, + } + descriptor := ooapi.NewSubmitMeasurementDescriptor(req, m.ReportID) + _, err := httpapi.Call(ctx, descriptor, c.endpoint) + return err +} diff --git a/internal/cmd/dismantle/main.go b/internal/cmd/dismantle/main.go new file mode 100644 index 0000000000..43f12b26a5 --- /dev/null +++ b/internal/cmd/dismantle/main.go @@ -0,0 +1,155 @@ +package main + +import ( + "context" + "fmt" + "net/url" + "os" + "path/filepath" + "time" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/backendclient" + "github.com/ooni/probe-cli/v3/internal/bytecounter" + "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivitylte" + "github.com/ooni/probe-cli/v3/internal/geolocate" + "github.com/ooni/probe-cli/v3/internal/kvstore" + "github.com/ooni/probe-cli/v3/internal/logx" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/platform" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/sessionhttpclient" + "github.com/ooni/probe-cli/v3/internal/sessionresolver" + "github.com/ooni/probe-cli/v3/internal/tunnel" + "github.com/ooni/probe-cli/v3/internal/version" +) + +func main() { + const softwareName = "dismantle" + const softwareVersion = "0.1.0-dev" + userAgent := fmt.Sprintf( + "%s/%s ooniprobe-engine/%s", + softwareName, softwareVersion, + version.Version, + ) + + logHandler := logx.NewHandlerWithDefaultSettings() + logHandler.Emoji = true + logger := &log.Logger{Level: log.InfoLevel, Handler: logHandler} + progressBar := model.NewPrinterCallbacks(logger) + counter := bytecounter.New() + home := filepath.Join(os.Getenv("HOME"), ".miniooni") + statedir := filepath.Join(home, "kvstore2") + ctx := context.Background() + tunnelDir := filepath.Join(home, "tunnel") + runtimex.Try0(os.MkdirAll(tunnelDir, 0700)) + + kvstore := runtimex.Try1(kvstore.NewFS(statedir)) + + tunnelConfig := &tunnel.Config{ + Name: "tor", + TunnelDir: tunnelDir, + Logger: logger, + } + tunnel, _ := runtimex.Try2(tunnel.Start(ctx, tunnelConfig)) + defer tunnel.Stop() + proxyURL := tunnel.SOCKS5ProxyURL() + + sessionResolver := &sessionresolver.Resolver{ + ByteCounter: counter, + KVStore: kvstore, + Logger: logger, + ProxyURL: proxyURL, + } + defer sessionResolver.CloseIdleConnections() + + geolocateConfig := geolocate.Config{ + Resolver: sessionResolver, + Logger: logger, + UserAgent: model.HTTPHeaderUserAgent, + } + geolocateTask := geolocate.NewTask(geolocateConfig) // XXX + location := runtimex.Try1(geolocateTask.Run(ctx)) + logger.Infof("%+v", location) + + sessionHTTPClientConfig := &sessionhttpclient.Config{ + ByteCounter: counter, + Logger: logger, + Resolver: sessionResolver, + ProxyURL: proxyURL, + } + sessionHTTPClient := sessionhttpclient.New(sessionHTTPClientConfig) + defer sessionHTTPClient.CloseIdleConnections() + + backendClientConfig := &backendclient.Config{ + KVStore: kvstore, + HTTPClient: sessionHTTPClient, + Logger: logger, + UserAgent: userAgent, + BaseURL: nil, + } + backendClient := backendclient.New(backendClientConfig) + + checkInConfig := &model.OOAPICheckInConfig{ + Charging: false, + OnWiFi: false, + Platform: platform.Name(), + ProbeASN: location.ProbeASNString(), + ProbeCC: location.CountryCode, + RunType: "manual", + SoftwareName: softwareName, + SoftwareVersion: softwareName, + WebConnectivity: model.OOAPICheckInConfigWebConnectivity{ + CategoryCodes: []string{}, + }, + } + checkInResult := runtimex.Try1(backendClient.CheckIn(ctx, checkInConfig)) + logger.Infof("%+v", checkInResult) + + runtimex.Assert(checkInResult.Tests.WebConnectivity != nil, "no web connectivity info") + reportID := checkInResult.Tests.WebConnectivity.ReportID + + experimentSession := &experimentSession{ + httpClient: sessionHTTPClient, + location: location, + logger: logger, + testHelpers: checkInResult.Conf.TestHelpers, + userAgent: userAgent, + } + + testStartTime := time.Now() + for _, input := range checkInResult.Tests.WebConnectivity.URLs { + cfg := &webconnectivitylte.Config{} + runner := webconnectivitylte.NewExperimentMeasurer(cfg) + measurement := model.NewMeasurement( + location, runner.ExperimentName(), runner.ExperimentVersion(), + testStartTime, reportID, softwareName, softwareVersion, input.URL, + ) + args := &model.ExperimentArgs{ + Callbacks: progressBar, + Measurement: measurement, + Session: experimentSession, + } + if err := runner.Run(ctx, args); err != nil { + logger.Warnf("runner.Run failed: %s", err.Error()) + } + if err := backendClient.Submit(ctx, measurement); err != nil { + logger.Warnf("backendClient.Submit failed: %s", err.Error()) + } + log.Infof("measurement URL: %s", makeExplorerURL(reportID, input.URL)) + } +} + +func makeExplorerURL(reportID, input string) string { + query := url.Values{} + query.Add("input", input) + explorerURL := &url.URL{ + Scheme: "https", + Host: "explorer.ooni.org", + Path: fmt.Sprintf("/measurement/%s", reportID), + RawQuery: query.Encode(), + Fragment: "", + RawFragment: "", + } + return explorerURL.String() +} diff --git a/internal/cmd/dismantle/session.go b/internal/cmd/dismantle/session.go new file mode 100644 index 0000000000..3806e508ba --- /dev/null +++ b/internal/cmd/dismantle/session.go @@ -0,0 +1,81 @@ +package main + +import ( + "context" + + "github.com/ooni/probe-cli/v3/internal/geolocate" + "github.com/ooni/probe-cli/v3/internal/model" +) + +type experimentSession struct { + httpClient model.HTTPClient + location *geolocate.Results + logger model.Logger + testHelpers map[string][]model.OOAPIService + userAgent string +} + +var _ model.ExperimentSession = &experimentSession{} + +// DefaultHTTPClient implements model.ExperimentSession +func (es *experimentSession) DefaultHTTPClient() model.HTTPClient { + return es.httpClient +} + +// FetchPsiphonConfig implements model.ExperimentSession +func (es *experimentSession) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { + // FIXME: we need to call the backend API for this I think? + panic("unimplemented") +} + +// FetchTorTargets implements model.ExperimentSession +func (es *experimentSession) FetchTorTargets(ctx context.Context, cc string) (map[string]model.OOAPITorTarget, error) { + // FIXME: we need to call the backend API for this I think? + panic("unimplemented") +} + +// GetTestHelpersByName implements model.ExperimentSession +func (es *experimentSession) GetTestHelpersByName(name string) ([]model.OOAPIService, bool) { + value, found := es.testHelpers[name] + return value, found +} + +// Logger implements model.ExperimentSession +func (es *experimentSession) Logger() model.Logger { + return es.logger +} + +// ProbeCC implements model.ExperimentSession +func (es *experimentSession) ProbeCC() string { + return es.location.CountryCode +} + +// ResolverIP implements model.ExperimentSession +func (es *experimentSession) ResolverIP() string { + return es.location.ResolverIPAddr +} + +// TempDir implements model.ExperimentSession +func (es *experimentSession) TempDir() string { + panic("unimplemented") // FIXME +} + +// TorArgs implements model.ExperimentSession +func (es *experimentSession) TorArgs() []string { + panic("unimplemented") // FIXME +} + +// TorBinary implements model.ExperimentSession +func (es *experimentSession) TorBinary() string { + panic("unimplemented") // FIXME +} + +// TunnelDir implements model.ExperimentSession +func (es *experimentSession) TunnelDir() string { + panic("unimplemented") // FIXME +} + +// UserAgent implements model.ExperimentSession +func (es *experimentSession) UserAgent() string { + return es.userAgent +} diff --git a/internal/cmd/dismantle2/client.go b/internal/cmd/dismantle2/client.go new file mode 100644 index 0000000000..422428a3d2 --- /dev/null +++ b/internal/cmd/dismantle2/client.go @@ -0,0 +1,173 @@ +package main + +import ( + "context" + "sync" + + "github.com/ooni/probe-cli/v3/internal/geolocate" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/session" +) + +// Client is a client for a [Session]. You should use a [Client] +// when you are writing Go code that uses a [Session]. +type Client struct { + logger model.Logger + once sync.Once + session *session.Session +} + +// NewClient creates a [Client] instance. +func NewClient(logger model.Logger) *Client { + return &Client{ + logger: logger, + once: sync.Once{}, + session: session.New(), + } +} + +// Bootstrap bootstraps a session. +func (c *Client) Bootstrap(ctx context.Context, req *session.BootstrapRequest) error { + if err := c.session.Send(ctx, &session.Request{Bootstrap: req}); err != nil { + return err + } + for { + resp, err := c.session.Recv(ctx) + if err != nil { + return err + } + if resp.Log != nil { + c.emitLog(resp.Log) + continue + } + if resp.Ticker != nil { + c.logger.Infof("bootstrap in progress (elapsed: %+v)", resp.Ticker.ElapsedTime) + continue + } + if resp.Bootstrap != nil { + return resp.Bootstrap.Error + } + c.logger.Warnf("unexpected event: %+v", resp) + } +} + +// Geolocate performs geolocation. You must run bootstrap first. +func (c *Client) Geolocate(ctx context.Context, req *session.GeolocateRequest) (*geolocate.Results, error) { + if err := c.session.Send(ctx, &session.Request{Geolocate: req}); err != nil { + return nil, err + } + for { + resp, err := c.session.Recv(ctx) + if err != nil { + return nil, err + } + if resp.Log != nil { + c.emitLog(resp.Log) + continue + } + if resp.Ticker != nil { + c.logger.Infof("geolocate in progress (elapsed: %+v)", resp.Ticker.ElapsedTime) + continue + } + if resp.Geolocate != nil { + return resp.Geolocate.Location, resp.Geolocate.Error + } + c.logger.Warnf("unexpected event: %+v", resp) + } +} + +// CheckIn calls the check-in API. You must run bootstrap first. +func (c *Client) CheckIn(ctx context.Context, req *session.CheckInRequest) (*model.OOAPICheckInResult, error) { + if err := c.session.Send(ctx, &session.Request{CheckIn: req}); err != nil { + return nil, err + } + for { + resp, err := c.session.Recv(ctx) + if err != nil { + return nil, err + } + if resp.Log != nil { + c.emitLog(resp.Log) + continue + } + if resp.Ticker != nil { + c.logger.Infof("check-in in progress (elapsed: %+v)", resp.Ticker.ElapsedTime) + continue + } + if resp.CheckIn != nil { + return resp.CheckIn.Result, resp.CheckIn.Error + } + c.logger.Warnf("unexpected event: %+v", resp) + } +} + +// WebConnectivity runs a single-URL Web Connectivity measurement. +func (c *Client) WebConnectivity( + ctx context.Context, req *session.WebConnectivityRequest) (*model.Measurement, error) { + if err := c.session.Send(ctx, &session.Request{WebConnectivity: req}); err != nil { + return nil, err + } + for { + resp, err := c.session.Recv(ctx) + if err != nil { + return nil, err + } + if resp.Log != nil { + c.emitLog(resp.Log) + continue + } + if resp.Ticker != nil { + c.logger.Infof("webconnectivity (elapsed: %+v)", resp.Ticker.ElapsedTime) + continue + } + if resp.WebConnectivity != nil { + return resp.WebConnectivity.Measurement, resp.WebConnectivity.Error + } + c.logger.Warnf("unexpected event: %+v", resp) + } +} + +// Submit submits a measurement. +func (c *Client) Submit(ctx context.Context, measurement *model.Measurement) error { + if err := c.session.Send(ctx, &session.Request{Submit: measurement}); err != nil { + return err + } + for { + resp, err := c.session.Recv(ctx) + if err != nil { + return err + } + if resp.Log != nil { + c.emitLog(resp.Log) + continue + } + if resp.Ticker != nil { + c.logger.Infof("webconnectivity (elapsed: %+v)", resp.Ticker.ElapsedTime) + continue + } + if resp.Submit != nil { + return resp.Submit.Error + } + c.logger.Warnf("unexpected event: %+v", resp) + } +} + +// Close releases the resources used by a [Client]. +func (c *Client) Close() (err error) { + c.once.Do(func() { + err = c.session.Close() + }) + return +} + +// emitLog emits a log event. +func (c *Client) emitLog(ev *session.LogEvent) { + switch ev.Level { + case "DEBUG": + c.logger.Debug(ev.Message) + case "WARNING": + c.logger.Warn(ev.Message) + default: + c.logger.Info(ev.Message) + } +} diff --git a/internal/cmd/dismantle2/main.go b/internal/cmd/dismantle2/main.go new file mode 100644 index 0000000000..2c588664c3 --- /dev/null +++ b/internal/cmd/dismantle2/main.go @@ -0,0 +1,79 @@ +package main + +import ( + "context" + "path/filepath" + "time" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/platform" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/session" +) + +const softwareName = "dismantle" + +const softwareVersion = "0.1.0-dev" + +func main() { + log.SetLevel(log.DebugLevel) + + client := NewClient(log.Log) + ctx := context.Background() + + bootstrapRequest := &session.BootstrapRequest{ + SnowflakeRendezvousMethod: "", + StateDir: filepath.Join("testdata", "engine"), + ProxyURL: "psiphon:///", + SoftwareName: softwareName, + SoftwareVersion: softwareVersion, + TorArgs: nil, + TorBinary: "", + TempDir: filepath.Join("testdata", "tmp"), + TunnelDir: filepath.Join("testdata", "tunnel"), + VerboseLogging: false, + } + runtimex.Try0(client.Bootstrap(ctx, bootstrapRequest)) + + geolocateRequest := &session.GeolocateRequest{} + location := runtimex.Try1(client.Geolocate(ctx, geolocateRequest)) + log.Infof("%+v", location) + + checkInRequest := &session.CheckInRequest{ + Charging: true, + OnWiFi: true, + Platform: platform.Name(), + ProbeASN: location.ProbeASNString(), + ProbeCC: location.ProbeCC(), + RunType: "manual", + SoftwareName: softwareName, + SoftwareVersion: softwareVersion, + WebConnectivity: model.OOAPICheckInConfigWebConnectivity{ + CategoryCodes: []string{}, + }, + } + checkInResponse := runtimex.Try1(client.CheckIn(ctx, checkInRequest)) + log.Infof("%+v", checkInResponse) + + runtimex.Assert(checkInResponse.Tests.WebConnectivity != nil, "no web connectivity info") + + reportID := checkInResponse.Tests.WebConnectivity.ReportID + testStartTime := time.Now() + for _, entry := range checkInResponse.Tests.WebConnectivity.URLs { + webConnectivityRequest := &session.WebConnectivityRequest{ + Input: entry.URL, + ReportID: reportID, + TestStartTime: testStartTime, + } + measurement, err := client.WebConnectivity(ctx, webConnectivityRequest) + if err != nil { + log.Warnf("webconnectivity: measure: %s", err.Error()) + continue + } + if err := client.Submit(ctx, measurement); err != nil { + log.Warnf("webconnectivity: submit: %s", err.Error()) + continue + } + } +} diff --git a/internal/cmd/tortunnel/main.go b/internal/cmd/tortunnel/main.go new file mode 100644 index 0000000000..f6c5d59148 --- /dev/null +++ b/internal/cmd/tortunnel/main.go @@ -0,0 +1,20 @@ +package main + +import ( + "context" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/tortunnel" +) + +func main() { + config := &tortunnel.Config{ + Logger: log.Log, + } + ctx := context.Background() + tunnel, err := tortunnel.Start(ctx, config) + if err != nil { + log.Fatalf("failure: %s", err.Error()) + } + tunnel.Stop() +} diff --git a/internal/database/actions_test.go b/internal/database/actions_test.go index 390ecf6d25..bbadfecc98 100644 --- a/internal/database/actions_test.go +++ b/internal/database/actions_test.go @@ -20,6 +20,8 @@ type locationInfo struct { resolverIP string } +var _ model.LocationProvider = &locationInfo{} + func (lp *locationInfo) ProbeASN() uint { return lp.asn } @@ -44,6 +46,21 @@ func (lp *locationInfo) ResolverIP() string { return lp.resolverIP } +// ResolverASN implements model.LocationProvider +func (*locationInfo) ResolverASN() uint { + panic("unimplemented") +} + +// ResolverASNString implements model.LocationProvider +func (*locationInfo) ResolverASNString() string { + panic("unimplemented") +} + +// ResolverNetworkName implements model.LocationProvider +func (*locationInfo) ResolverNetworkName() string { + panic("unimplemented") +} + func TestNewDatabase(t *testing.T) { t.Run("with empty path", func(t *testing.T) { dbpath := "" diff --git a/internal/engine/experiment_test.go b/internal/engine/experiment_test.go index 84298039fb..740b453335 100644 --- a/internal/engine/experiment_test.go +++ b/internal/engine/experiment_test.go @@ -24,7 +24,7 @@ func TestExperimentHonoursSharingDefaults(t *testing.T) { } allspecs := []spec{{ name: "probeIP", - locationInfo: &geolocate.Results{ProbeIP: "8.8.8.8"}, + locationInfo: &geolocate.Results{IPAddr: "8.8.8.8"}, expect: func(m *model.Measurement) bool { return m.ProbeIP == model.DefaultProbeIP }, @@ -48,19 +48,19 @@ func TestExperimentHonoursSharingDefaults(t *testing.T) { }, }, { name: "resolverIP", - locationInfo: &geolocate.Results{ResolverIP: "9.9.9.9"}, + locationInfo: &geolocate.Results{ResolverIPAddr: "9.9.9.9"}, expect: func(m *model.Measurement) bool { return m.ResolverIP == "9.9.9.9" }, }, { name: "resolverASN", - locationInfo: &geolocate.Results{ResolverASN: 44}, + locationInfo: &geolocate.Results{ResolverASNumber: 44}, expect: func(m *model.Measurement) bool { return m.ResolverASN == "AS44" }, }, { name: "resolverNetworkName", - locationInfo: &geolocate.Results{ResolverNetworkName: "Google LLC"}, + locationInfo: &geolocate.Results{ResolverASNetworkName: "Google LLC"}, expect: func(m *model.Measurement) bool { return m.ResolverNetworkName == "Google LLC" }, diff --git a/internal/engine/session.go b/internal/engine/session.go index 3169d711ca..a8249b69d2 100644 --- a/internal/engine/session.go +++ b/internal/engine/session.go @@ -191,7 +191,7 @@ func NewSession(ctx context.Context, config SessionConfig) (*Session, error) { Logger: config.Logger, Name: proxyURL.Scheme, SnowflakeRendezvous: config.SnowflakeRendezvous, - Session: &sessionTunnelEarlySession{}, + Session: &SessionTunnelEarlySession{}, TorArgs: config.TorArgs, TorBinary: config.TorBinary, TunnelDir: config.TunnelDir, @@ -523,7 +523,7 @@ func (s *Session) ProbeIP() string { s.mu.Lock() ip := model.DefaultProbeIP if s.location != nil { - ip = s.location.ProbeIP + ip = s.location.IPAddr } return ip } @@ -544,7 +544,7 @@ func (s *Session) ResolverASN() uint { s.mu.Lock() asn := model.DefaultResolverASN if s.location != nil { - asn = s.location.ResolverASN + asn = s.location.ResolverASNumber } return asn } @@ -555,7 +555,7 @@ func (s *Session) ResolverIP() string { s.mu.Lock() ip := model.DefaultResolverIP if s.location != nil { - ip = s.location.ResolverIP + ip = s.location.ResolverIPAddr } return ip } @@ -566,7 +566,7 @@ func (s *Session) ResolverNetworkName() string { s.mu.Lock() nn := model.DefaultResolverNetworkName if s.location != nil { - nn = s.location.ResolverNetworkName + nn = s.location.ResolverASNetworkName } return nn } diff --git a/internal/engine/session_nopsiphon.go b/internal/engine/session_nopsiphon.go index cc03326d57..16683ae5ea 100644 --- a/internal/engine/session_nopsiphon.go +++ b/internal/engine/session_nopsiphon.go @@ -16,16 +16,16 @@ func (s *Session) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { return clnt.FetchPsiphonConfig(ctx) } -// sessionTunnelEarlySession is the early session that we pass +// SessionTunnelEarlySession is the early session that we pass // to tunnel.Start to fetch the Psiphon configuration. -type sessionTunnelEarlySession struct{} +type SessionTunnelEarlySession struct{} // errPsiphonNoEmbeddedConfig indicates that there is no // embedded psiphong config file in this binary. var errPsiphonNoEmbeddedConfig = errors.New("no embedded configuration file") // FetchPsiphonConfig implements tunnel.Session.FetchPsiphonConfig. -func (s *sessionTunnelEarlySession) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { +func (s *SessionTunnelEarlySession) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { return nil, errPsiphonNoEmbeddedConfig } diff --git a/internal/engine/session_nopsiphon_test.go b/internal/engine/session_nopsiphon_test.go index ecb6cc75a4..99e013e7da 100644 --- a/internal/engine/session_nopsiphon_test.go +++ b/internal/engine/session_nopsiphon_test.go @@ -9,7 +9,7 @@ import ( ) func TestEarlySessionNoPsiphonFetchPsiphonConfig(t *testing.T) { - s := &sessionTunnelEarlySession{} + s := &SessionTunnelEarlySession{} out, err := s.FetchPsiphonConfig(context.Background()) if !errors.Is(err, errPsiphonNoEmbeddedConfig) { t.Fatal("not the error we expected", err) diff --git a/internal/engine/session_psiphon.go b/internal/engine/session_psiphon.go index 48b5ab0a06..bd4651761a 100644 --- a/internal/engine/session_psiphon.go +++ b/internal/engine/session_psiphon.go @@ -17,13 +17,13 @@ var psiphonConfigJSONAge []byte //go:embed psiphon-config.key var psiphonConfigSecretKey string -// sessionTunnelEarlySession is the early session that we pass +// SessionTunnelEarlySession is the early session that we pass // to tunnel.Start to fetch the Psiphon configuration. -type sessionTunnelEarlySession struct{} +type SessionTunnelEarlySession struct{} // FetchPsiphonConfig decrypts psiphonConfigJSONAge using // filippo.io/age _and_ psiphonConfigSecretKey. -func (s *sessionTunnelEarlySession) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { +func (s *SessionTunnelEarlySession) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { key := "AGE-SECRET-KEY-1" + psiphonConfigSecretKey identity, err := age.ParseX25519Identity(key) if err != nil { @@ -40,13 +40,13 @@ func (s *sessionTunnelEarlySession) FetchPsiphonConfig(ctx context.Context) ([]b // FetchPsiphonConfig decrypts psiphonConfigJSONAge using // filippo.io/age _and_ psiphonConfigSecretKey. func (s *Session) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { - child := &sessionTunnelEarlySession{} + child := &SessionTunnelEarlySession{} return child.FetchPsiphonConfig(ctx) } // CheckEmbeddedPsiphonConfig checks whether we can load psiphon's config func CheckEmbeddedPsiphonConfig() error { - child := &sessionTunnelEarlySession{} + child := &SessionTunnelEarlySession{} _, err := child.FetchPsiphonConfig(context.Background()) return err } diff --git a/internal/experiment/webconnectivity/webconnectivity.go b/internal/experiment/webconnectivity/webconnectivity.go index 40732abcb9..abd8c76b2a 100644 --- a/internal/experiment/webconnectivity/webconnectivity.go +++ b/internal/experiment/webconnectivity/webconnectivity.go @@ -14,8 +14,11 @@ import ( ) const ( - testName = "web_connectivity" - testVersion = "0.4.2" + // ExperimentName is the experiment name + ExperimentName = "web_connectivity" + + // ExperimentVersion is the experiment version + ExperimentVersion = "0.4.2" ) // Config contains the experiment config. @@ -85,12 +88,12 @@ func NewExperimentMeasurer(config Config) model.ExperimentMeasurer { // ExperimentName implements ExperimentMeasurer.ExperExperimentName. func (m Measurer) ExperimentName() string { - return testName + return ExperimentName } // ExperimentVersion implements ExperimentMeasurer.ExperExperimentVersion. func (m Measurer) ExperimentVersion() string { - return testVersion + return ExperimentVersion } var ( @@ -261,6 +264,11 @@ type SummaryKeys struct { // GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) { + return GetSummaryKeys(measurement) +} + +// GetSummaryKeys returns the Web Connectivity summary keys +func GetSummaryKeys(measurement *model.Measurement) (interface{}, error) { sk := SummaryKeys{IsAnomaly: false} tk, ok := measurement.TestKeys.(*TestKeys) if !ok { diff --git a/internal/geolocate/cloudflare.go b/internal/geolocate/cloudflare.go index 3de61a538b..4cbc611f42 100644 --- a/internal/geolocate/cloudflare.go +++ b/internal/geolocate/cloudflare.go @@ -28,6 +28,5 @@ func cloudflareIPLookup( } r := regexp.MustCompile("(?:ip)=(.*)") ip := strings.Trim(string(r.Find(data)), "ip=") - logger.Debugf("cloudflare: body: %s", ip) return ip, nil } diff --git a/internal/geolocate/geolocate.go b/internal/geolocate/geolocate.go index a9c6f5f23d..9cf8c300f0 100644 --- a/internal/geolocate/geolocate.go +++ b/internal/geolocate/geolocate.go @@ -24,24 +24,66 @@ type Results struct { // NetworkName is the network name. NetworkName string - // IP is the probe IP. - ProbeIP string + // IPAddr is the probe IPAddr. + IPAddr string - // ResolverASN is the resolver ASN. - ResolverASN uint + // ResolverASNumber is the resolver ASN. + ResolverASNumber uint - // ResolverIP is the resolver IP. - ResolverIP string + // ResolverIPAddr is the resolver IP. + ResolverIPAddr string - // ResolverNetworkName is the resolver network name. - ResolverNetworkName string + // ResolverASNetworkName is the resolver network name. + ResolverASNetworkName string } -// ASNString returns the ASN as a string. -func (r *Results) ASNString() string { +var _ model.LocationProvider = &Results{} + +// ProbeASN implements model.LocationProvider +func (r *Results) ProbeASN() uint { + return r.ASN +} + +// ProbeASNString returns the ASN as a string. +func (r *Results) ProbeASNString() string { return fmt.Sprintf("AS%d", r.ASN) } +// ProbeCC implements model.LocationProvider +func (r *Results) ProbeCC() string { + return r.CountryCode +} + +// ProbeIP implements model.LocationProvider +func (r *Results) ProbeIP() string { + return r.IPAddr +} + +// ProbeNetworkName implements model.LocationProvider +func (r *Results) ProbeNetworkName() string { + return r.NetworkName +} + +// ResolverASN implements model.LocationProvider +func (r *Results) ResolverASN() uint { + return r.ResolverASNumber +} + +// ResolverASNString implements model.LocationProvider +func (r *Results) ResolverASNString() string { + return fmt.Sprintf("AS%d", r.ResolverASNumber) +} + +// ResolverIP implements model.LocationProvider +func (r *Results) ResolverIP() string { + return r.ResolverIPAddr +} + +// ResolverNetworkName implements model.LocationProvider +func (r *Results) ResolverNetworkName() string { + return r.ResolverASNetworkName +} + type probeIPLookupper interface { LookupProbeIP(ctx context.Context) (addr string, err error) } @@ -87,6 +129,7 @@ func NewTask(config Config) *Task { } return &Task{ countryLookupper: mmdbLookupper{}, + logger: config.Logger, probeIPLookupper: ipLookupClient(config), probeASNLookupper: mmdbLookupper{}, resolverASNLookupper: mmdbLookupper{}, @@ -100,6 +143,7 @@ func NewTask(config Config) *Task { // instance of Task using the NewTask factory. type Task struct { countryLookupper countryLookupper + logger model.Logger probeIPLookupper probeIPLookupper probeASNLookupper asnLookupper resolverASNLookupper asnLookupper @@ -110,29 +154,32 @@ type Task struct { func (op Task) Run(ctx context.Context) (*Results, error) { var err error out := &Results{ - ASN: model.DefaultProbeASN, - CountryCode: model.DefaultProbeCC, - NetworkName: model.DefaultProbeNetworkName, - ProbeIP: model.DefaultProbeIP, - ResolverASN: model.DefaultResolverASN, - ResolverIP: model.DefaultResolverIP, - ResolverNetworkName: model.DefaultResolverNetworkName, + ASN: model.DefaultProbeASN, + CountryCode: model.DefaultProbeCC, + NetworkName: model.DefaultProbeNetworkName, + IPAddr: model.DefaultProbeIP, + ResolverASNumber: model.DefaultResolverASN, + ResolverIPAddr: model.DefaultResolverIP, + ResolverASNetworkName: model.DefaultResolverNetworkName, } ip, err := op.probeIPLookupper.LookupProbeIP(ctx) if err != nil { return out, fmt.Errorf("lookupProbeIP failed: %w", err) } - out.ProbeIP = ip - asn, networkName, err := op.probeASNLookupper.LookupASN(out.ProbeIP) + out.IPAddr = ip + asn, networkName, err := op.probeASNLookupper.LookupASN(out.IPAddr) if err != nil { return out, fmt.Errorf("lookupASN failed: %w", err) } + op.logger.Infof("geolocate: probe ASN: %d", asn) + op.logger.Infof("geolocate: probe network name: %s", networkName) out.ASN = asn out.NetworkName = networkName - cc, err := op.countryLookupper.LookupCC(out.ProbeIP) + cc, err := op.countryLookupper.LookupCC(out.IPAddr) if err != nil { return out, fmt.Errorf("lookupProbeCC failed: %w", err) } + op.logger.Infof("geolocate: country code: %s", cc) out.CountryCode = cc out.didResolverLookup = true // Note: ignoring the result of lookupResolverIP and lookupASN @@ -143,14 +190,16 @@ func (op Task) Run(ctx context.Context) (*Results, error) { if err != nil { return out, nil // intentional } - out.ResolverIP = resolverIP + out.ResolverIPAddr = resolverIP resolverASN, resolverNetworkName, err := op.resolverASNLookupper.LookupASN( - out.ResolverIP, + out.ResolverIPAddr, ) if err != nil { return out, nil // intentional } - out.ResolverASN = resolverASN - out.ResolverNetworkName = resolverNetworkName + op.logger.Infof("geolocate: resolver ASN: %d", resolverASN) + op.logger.Infof("geolocate: resolver network name: %s", resolverNetworkName) + out.ResolverASNumber = resolverASN + out.ResolverASNetworkName = resolverNetworkName return out, nil } diff --git a/internal/geolocate/geolocate_test.go b/internal/geolocate/geolocate_test.go index 9d10917531..004d83fc44 100644 --- a/internal/geolocate/geolocate_test.go +++ b/internal/geolocate/geolocate_test.go @@ -36,16 +36,16 @@ func TestLocationLookupCannotLookupProbeIP(t *testing.T) { if out.NetworkName != model.DefaultProbeNetworkName { t.Fatal("invalid NetworkName value") } - if out.ProbeIP != model.DefaultProbeIP { + if out.IPAddr != model.DefaultProbeIP { t.Fatal("invalid ProbeIP value") } - if out.ResolverASN != model.DefaultResolverASN { + if out.ResolverASNumber != model.DefaultResolverASN { t.Fatal("invalid ResolverASN value") } - if out.ResolverIP != model.DefaultResolverIP { + if out.ResolverIPAddr != model.DefaultResolverIP { t.Fatal("invalid ResolverIP value") } - if out.ResolverNetworkName != model.DefaultResolverNetworkName { + if out.ResolverASNetworkName != model.DefaultResolverNetworkName { t.Fatal("invalid ResolverNetworkName value") } } @@ -80,16 +80,16 @@ func TestLocationLookupCannotLookupProbeASN(t *testing.T) { if out.NetworkName != model.DefaultProbeNetworkName { t.Fatal("invalid NetworkName value") } - if out.ProbeIP != "1.2.3.4" { + if out.IPAddr != "1.2.3.4" { t.Fatal("invalid ProbeIP value") } - if out.ResolverASN != model.DefaultResolverASN { + if out.ResolverASNumber != model.DefaultResolverASN { t.Fatal("invalid ResolverASN value") } - if out.ResolverIP != model.DefaultResolverIP { + if out.ResolverIPAddr != model.DefaultResolverIP { t.Fatal("invalid ResolverIP value") } - if out.ResolverNetworkName != model.DefaultResolverNetworkName { + if out.ResolverASNetworkName != model.DefaultResolverNetworkName { t.Fatal("invalid ResolverNetworkName value") } } @@ -124,16 +124,16 @@ func TestLocationLookupCannotLookupProbeCC(t *testing.T) { if out.NetworkName != "1234.com" { t.Fatal("invalid NetworkName value") } - if out.ProbeIP != "1.2.3.4" { + if out.IPAddr != "1.2.3.4" { t.Fatal("invalid ProbeIP value") } - if out.ResolverASN != model.DefaultResolverASN { + if out.ResolverASNumber != model.DefaultResolverASN { t.Fatal("invalid ResolverASN value") } - if out.ResolverIP != model.DefaultResolverIP { + if out.ResolverIPAddr != model.DefaultResolverIP { t.Fatal("invalid ResolverIP value") } - if out.ResolverNetworkName != model.DefaultResolverNetworkName { + if out.ResolverASNetworkName != model.DefaultResolverNetworkName { t.Fatal("invalid ResolverNetworkName value") } } @@ -169,19 +169,19 @@ func TestLocationLookupCannotLookupResolverIP(t *testing.T) { if out.NetworkName != "1234.com" { t.Fatal("invalid NetworkName value") } - if out.ProbeIP != "1.2.3.4" { + if out.IPAddr != "1.2.3.4" { t.Fatal("invalid ProbeIP value") } if out.didResolverLookup != true { t.Fatal("invalid DidResolverLookup value") } - if out.ResolverASN != model.DefaultResolverASN { + if out.ResolverASNumber != model.DefaultResolverASN { t.Fatal("invalid ResolverASN value") } - if out.ResolverIP != model.DefaultResolverIP { + if out.ResolverIPAddr != model.DefaultResolverIP { t.Fatal("invalid ResolverIP value") } - if out.ResolverNetworkName != model.DefaultResolverNetworkName { + if out.ResolverASNetworkName != model.DefaultResolverNetworkName { t.Fatal("invalid ResolverNetworkName value") } } @@ -209,19 +209,19 @@ func TestLocationLookupCannotLookupResolverNetworkName(t *testing.T) { if out.NetworkName != "1234.com" { t.Fatal("invalid NetworkName value") } - if out.ProbeIP != "1.2.3.4" { + if out.IPAddr != "1.2.3.4" { t.Fatal("invalid ProbeIP value") } if out.didResolverLookup != true { t.Fatal("invalid DidResolverLookup value") } - if out.ResolverASN != model.DefaultResolverASN { - t.Fatalf("invalid ResolverASN value: %+v", out.ResolverASN) + if out.ResolverASNumber != model.DefaultResolverASN { + t.Fatalf("invalid ResolverASN value: %+v", out.ResolverASNumber) } - if out.ResolverIP != "4.3.2.1" { - t.Fatalf("invalid ResolverIP value: %+v", out.ResolverIP) + if out.ResolverIPAddr != "4.3.2.1" { + t.Fatalf("invalid ResolverIP value: %+v", out.ResolverIPAddr) } - if out.ResolverNetworkName != model.DefaultResolverNetworkName { + if out.ResolverASNetworkName != model.DefaultResolverNetworkName { t.Fatal("invalid ResolverNetworkName value") } } @@ -248,19 +248,19 @@ func TestLocationLookupSuccessWithResolverLookup(t *testing.T) { if out.NetworkName != "1234.com" { t.Fatal("invalid NetworkName value") } - if out.ProbeIP != "1.2.3.4" { + if out.IPAddr != "1.2.3.4" { t.Fatal("invalid ProbeIP value") } if out.didResolverLookup != true { t.Fatal("invalid DidResolverLookup value") } - if out.ResolverASN != 4321 { - t.Fatalf("invalid ResolverASN value: %+v", out.ResolverASN) + if out.ResolverASNumber != 4321 { + t.Fatalf("invalid ResolverASN value: %+v", out.ResolverASNumber) } - if out.ResolverIP != "4.3.2.1" { - t.Fatalf("invalid ResolverIP value: %+v", out.ResolverIP) + if out.ResolverIPAddr != "4.3.2.1" { + t.Fatalf("invalid ResolverIP value: %+v", out.ResolverIPAddr) } - if out.ResolverNetworkName != "4321.com" { + if out.ResolverASNetworkName != "4321.com" { t.Fatal("invalid ResolverNetworkName value") } } @@ -281,7 +281,7 @@ func TestSmoke(t *testing.T) { func TestASNStringWorks(t *testing.T) { r := Results{ASN: 1234} - if r.ASNString() != "AS1234" { + if r.ProbeASNString() != "AS1234" { t.Fatal("unexpected result") } } diff --git a/internal/geolocate/iplookup.go b/internal/geolocate/iplookup.go index 3d9711f323..47b23a55c4 100644 --- a/internal/geolocate/iplookup.go +++ b/internal/geolocate/iplookup.go @@ -97,14 +97,14 @@ func (c ipLookupClient) doWithCustomFunc( if net.ParseIP(ip) == nil { return model.DefaultProbeIP, fmt.Errorf("%w: %s", ErrInvalidIPAddress, ip) } - c.Logger.Debugf("iplookup: IP: %s", ip) + c.Logger.Debugf("geolocate: IP: %s", ip) return ip, nil } func (c ipLookupClient) LookupProbeIP(ctx context.Context) (string, error) { union := multierror.New(ErrAllIPLookuppersFailed) for _, method := range makeSlice() { - c.Logger.Infof("iplookup: using %s", method.name) + c.Logger.Infof("geolocate: using %s", method.name) ip, err := c.doWithCustomFunc(ctx, method.fn) if err == nil { return ip, nil diff --git a/internal/geolocate/ubuntu.go b/internal/geolocate/ubuntu.go index 0f75afb4e0..2556470d4f 100644 --- a/internal/geolocate/ubuntu.go +++ b/internal/geolocate/ubuntu.go @@ -30,7 +30,6 @@ func ubuntuIPLookup( if err != nil { return model.DefaultProbeIP, err } - logger.Debugf("ubuntu: body: %s", string(data)) var v ubuntuResponse err = xml.Unmarshal(data, &v) if err != nil { diff --git a/internal/model/location.go b/internal/model/location.go index 36322f8abc..6c1735ae63 100644 --- a/internal/model/location.go +++ b/internal/model/location.go @@ -1,12 +1,61 @@ package model -// LocationProvider is an interface that returns the current location. The -// [engine.Session] struct implements this interface. +// LocationProvider is an interface that returns the current location. type LocationProvider interface { + // ProbeASN is the ASN associated with ProbeIP. ProbeASN() uint + + // ProbeASNString returns the probe ASN as the AS%d string. ProbeASNString() string + + // ProbeCC is the country code associated with ProbeIP. ProbeCC() string + + // ProbeIP is the probe IP address. ProbeIP() string + + // ProbeNetworkName is the name of the ProbeASN. ProbeNetworkName() string + + // ResolverIP is the IP of the resolver. ResolverIP() string + + // ResolverASN is the resolver ASN. + ResolverASN() uint + + // ResolverASNString is the resolver ASN as the AS%d string. + ResolverASNString() string + + // ResolverNetworkName is the name of the ResolverASN. + ResolverNetworkName() string +} + +// Location describes the probe's location. +type Location struct { + // ProbeASN is the ASN associated with ProbeIP. + ProbeASN int64 + + // ProbeASNString returns the probe ASN as the AS%d string. + ProbeASNString string + + // ProbeCC is the country code associated with ProbeIP. + ProbeCC string + + // ProbeIP is the probe IP address. + ProbeIP string + + // ProbeNetworkName is the name of the ProbeASN. + ProbeNetworkName string + + // ResolverIP is the IP of the resolver. + ResolverIP string + + // ResolverASN is the resolver ASN. + ResolverASN int64 + + // ResolverASNString is the resolver ASN as the AS%d string. + ResolverASNString string + + // ResolverNetworkName is the name of the ResolverASN. + ResolverNetworkName string } diff --git a/internal/model/measurement.go b/internal/model/measurement.go index 7cd2a8e13e..aad271da22 100644 --- a/internal/model/measurement.go +++ b/internal/model/measurement.go @@ -10,9 +10,12 @@ import ( "errors" "fmt" "net" + "runtime" "time" + "github.com/ooni/probe-cli/v3/internal/platform" "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/version" ) const ( @@ -215,3 +218,49 @@ func scrubTestKeys(m *Measurement, currentIP string) error { data = bytes.ReplaceAll(data, []byte(currentIP), []byte(Scrubbed)) return scrubJSONUnmarshalTestKeys(data, &m.TestKeys) } + +// dateFormat is the data format used to fill a measurement. +const dateFormat = "2006-01-02 15:04:05" + +// NewMeasurement constructs a new measurement. +func NewMeasurement( + location LocationProvider, + testName string, + testVersion string, + testStartTime time.Time, + reportID string, + softwareName string, + softwareVersion string, + input string, +) *Measurement { + utctimenow := time.Now().UTC() + m := &Measurement{ + DataFormatVersion: OOAPIReportDefaultDataFormatVersion, + Input: MeasurementTarget(input), + MeasurementStartTime: utctimenow.Format(dateFormat), + MeasurementStartTimeSaved: utctimenow, + ProbeIP: DefaultProbeIP, + ProbeASN: location.ProbeASNString(), + ProbeCC: location.ProbeCC(), + ProbeNetworkName: location.ProbeNetworkName(), + ReportID: reportID, + ResolverASN: location.ResolverASNString(), + ResolverIP: location.ResolverIP(), + ResolverNetworkName: location.ResolverNetworkName(), + SoftwareName: softwareName, + SoftwareVersion: softwareVersion, + TestName: testName, + TestStartTime: testStartTime.Format(dateFormat), + TestVersion: testVersion, + } + m.AddAnnotation("architecture", runtime.GOARCH) + m.AddAnnotation("engine_name", "ooniprobe-engine") + m.AddAnnotation("engine_version", version.Version) + m.AddAnnotation("go_version", runtimex.BuildInfo.GoVersion) + m.AddAnnotation("platform", platform.Name()) + m.AddAnnotation("vcs_modified", runtimex.BuildInfo.VcsModified) + m.AddAnnotation("vcs_revision", runtimex.BuildInfo.VcsRevision) + m.AddAnnotation("vcs_time", runtimex.BuildInfo.VcsTime) + m.AddAnnotation("vcs_tool", runtimex.BuildInfo.VcsTool) + return m +} diff --git a/internal/model/mocks/location.go b/internal/model/mocks/location.go index 2c0ecc7e6a..a48b71acf7 100644 --- a/internal/model/mocks/location.go +++ b/internal/model/mocks/location.go @@ -3,12 +3,15 @@ package mocks import "github.com/ooni/probe-cli/v3/internal/model" type LocationProvider struct { - MockProbeASN func() uint - MockProbeASNString func() string - MockProbeCC func() string - MockProbeIP func() string - MockProbeNetworkName func() string - MockResolverIP func() string + MockProbeASN func() uint + MockProbeASNString func() string + MockProbeCC func() string + MockProbeIP func() string + MockProbeNetworkName func() string + MockResolverIP func() string + MockResolverASN func() uint + MockResolverASNString func() string + MockResolverNetworkName func() string } var _ model.LocationProvider = &LocationProvider{} @@ -42,3 +45,18 @@ func (loc *LocationProvider) ProbeNetworkName() string { func (loc *LocationProvider) ResolverIP() string { return loc.MockResolverIP() } + +// ResolverASN implements model.LocationProvider +func (loc *LocationProvider) ResolverASN() uint { + return loc.MockResolverASN() +} + +// ResolverASNString implements model.LocationProvider +func (loc *LocationProvider) ResolverASNString() string { + return loc.MockResolverASNString() +} + +// ResolverNetworkName implements model.LocationProvider +func (loc *LocationProvider) ResolverNetworkName() string { + return loc.MockResolverNetworkName() +} diff --git a/internal/model/optional.go b/internal/model/optional.go new file mode 100644 index 0000000000..1eb2cf6410 --- /dev/null +++ b/internal/model/optional.go @@ -0,0 +1,33 @@ +package model + +import ( + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// OptionalPtr is an optional pointer. +type OptionalPtr[Type any] struct { + v *Type +} + +// NewOptionalPtr creates a new OptionalPtr. +func NewOptionalPtr[Type any](v *Type) OptionalPtr[Type] { + runtimex.Assert(v != nil, "passed nil pointer") + return OptionalPtr[Type]{v} +} + +// Unwrap returns the underlying pointer. This function panics +// if the underlying ptr is nil. +func (op OptionalPtr[Type]) Unwrap() *Type { + runtimex.Assert(op.IsSome(), "not initialized") + return op.v +} + +// IsSome returns whether the underlying ptr is not nil. +func (op OptionalPtr[Type]) IsSome() bool { + return op.v != nil +} + +// IsNone returns whether the underlying ptr is nil. +func (op OptionalPtr[Type]) IsNone() bool { + return !op.IsSome() +} diff --git a/internal/model/tunnel.go b/internal/model/tunnel.go new file mode 100644 index 0000000000..25d5def495 --- /dev/null +++ b/internal/model/tunnel.go @@ -0,0 +1,47 @@ +package model + +// +// Common interface for circumvention tunnels. +// + +import ( + "context" + "errors" +) + +// TunnelBootstrapEvent is an event emitted during the bootstrap. +type TunnelBootstrapEvent struct { + // Progress is the progress we have made so far as a number between 0 and 1. + Progress float64 + + // Message is the corresponding explanatory message. + Message string +} + +// Tunnel is a tunnel for communicating with the OONI backend. +type Tunnel interface { + // LookupProbeIP discovers the probe's IP address using this tunnel. + LookupProbeIP(ctx context.Context) (string, error) + + // Name returns the tunnel name. + Name() string + + // NewHTTPTransport returns a new HTTP transport using this tunnel. + NewHTTPTransport(logger Logger) (HTTPTransport, error) + + // NewDNSOverHTTPSResolver returns a new DNS-over-HTTPS resolver using this tunnel. + NewDNSOverHTTPSResolver(logger Logger, URL string) (Resolver, error) + + // Start starts the tunnel and returns two channels. The first channel gets + // interim bootstrap events, the second gets the final result. If the context + // is cancelled or expires during the bootstrap, we interrupt the bootstrap + // early and return an error via the error channel. If you already started + // the tunnel, this function posts a nil error on the second channel. + Start(ctx context.Context) (<-chan *TunnelBootstrapEvent, <-chan error) + + // Stop stops the tunnel. + Stop() +} + +// ErrTunnelNotStarted indicates we have not started the tunnel. +var ErrTunnelNotStarted = errors.New("tunnel: not started") diff --git a/internal/nettests/doc.go b/internal/nettests/doc.go new file mode 100644 index 0000000000..4c4482c063 --- /dev/null +++ b/internal/nettests/doc.go @@ -0,0 +1,2 @@ +// Package nettests contains the nettests implementation. +package nettests diff --git a/internal/nettests/experiment.go b/internal/nettests/experiment.go new file mode 100644 index 0000000000..7ca9c776ff --- /dev/null +++ b/internal/nettests/experiment.go @@ -0,0 +1,41 @@ +package nettests + +import ( + "errors" + + "github.com/ooni/probe-cli/v3/internal/model" + "golang.org/x/net/context" +) + +// ExperimentFactory constructs a generic [Experiment]. +type ExperimentFactory interface { + // NewExperiment creates a new instance of [Experiment]. + NewExperiment(callbacks model.ExperimentCallbacks) Experiment +} + +// Experiment is the generic API of all experiments. +type Experiment interface { + // GetSummaryKeys returns a data structure containing a + // summary of the test keys for ooniprobe. + GetSummaryKeys(m *model.Measurement) (any, error) + + // KibiBytesReceived accounts for the KibiBytes received by the experiment. + KibiBytesReceived() float64 + + // KibiBytesSent is like KibiBytesReceived but for the bytes sent. + KibiBytesSent() float64 + + // Measure performs a synchronous measurement and returns + // either a valid measurement or an error. + Measure(ctx context.Context, input string) (*model.Measurement, error) + + // Name returns the experiment name. + Name() string + + // ReportID returns the reportID used by this experiment. + ReportID() string +} + +// ErrMissingCheckInConfig is the error returned when we do not have +// a check-in configuration for the selected experiment. +var ErrMissingCheckInConfig = errors.New("nettests: missing check-in config") diff --git a/internal/nettests/inputloader.go b/internal/nettests/inputloader.go new file mode 100644 index 0000000000..d548e6bb50 --- /dev/null +++ b/internal/nettests/inputloader.go @@ -0,0 +1,77 @@ +package nettests + +import ( + "bufio" + "errors" + "fmt" + + "github.com/ooni/probe-cli/v3/internal/fsx" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// ErrDetectedEmptyFile indicates we detected an open file. +var ErrDetectedEmptyFile = errors.New("inputloader: file did not contain any input") + +// ErrNoAvailableInput indicates there's no available input. +var ErrNoAvailableInput = errors.New("inputloader: no available input") + +// loadInputs loads inputs by giving priority to user-defined +// input and falling back to API-provided inputs. +func loadInputs( + apiInputs []model.OOAPIURLInfo, + userFiles []string, + userInputs []string, +) ([]model.OOAPIURLInfo, error) { + inputs, err := loadLocalInputs(userFiles, userInputs) + if err != nil || len(inputs) > 0 { + return inputs, err + } + if len(apiInputs) <= 0 { + return nil, ErrNoAvailableInput + } + return apiInputs, nil +} + +// loadLocalInputs loads inputs from user-provided inputs and files. +func loadLocalInputs(userFiles, userInputs []string) ([]model.OOAPIURLInfo, error) { + inputs := []model.OOAPIURLInfo{} + for _, input := range userInputs { + inputs = append(inputs, model.OOAPIURLInfo{URL: input}) + } + for _, filepath := range userFiles { + extra, err := loadLocalInputFile(filepath) + if err != nil { + return nil, err + } + // See https://github.com/ooni/probe-engine/issues/1123. + if len(extra) <= 0 { + return nil, fmt.Errorf("%w: %s", ErrDetectedEmptyFile, filepath) + } + inputs = append(inputs, extra...) + } + return inputs, nil +} + +// loadLocalInputFile reads inputs from the specified file. +func loadLocalInputFile(filepath string) ([]model.OOAPIURLInfo, error) { + inputs := []model.OOAPIURLInfo{} + filep, err := fsx.OpenFile(filepath) + if err != nil { + return nil, err + } + defer filep.Close() + // Implementation note: when you save file with vim, you have newline at + // end of file and you don't want to consider that an input line. While there + // ignore any other empty line that may occur inside the file. + scanner := bufio.NewScanner(filep) + for scanner.Scan() { + line := scanner.Text() + if line != "" { + inputs = append(inputs, model.OOAPIURLInfo{URL: line}) + } + } + if scanner.Err() != nil { + return nil, scanner.Err() + } + return inputs, nil +} diff --git a/internal/nettests/session.go b/internal/nettests/session.go new file mode 100644 index 0000000000..d166b98f36 --- /dev/null +++ b/internal/nettests/session.go @@ -0,0 +1,142 @@ +package nettests + +import ( + "sync" + + "github.com/ooni/probe-cli/v3/internal/geolocate" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/session" + "golang.org/x/net/context" +) + +// Session is a measurement session. +type Session struct { + // bootstrapRequest contains settings to bootstrap a session. + bootstrapRequest *session.BootstrapRequest + + // logger is the logger to use. + logger model.Logger + + // once allows us to run cleanups just once. + once sync.Once + + // session is the initially empty session. + session *session.Session +} + +// NewSession creates a new [Session] instance. +func NewSession(request *session.BootstrapRequest, logger model.Logger) *Session { + return &Session{ + bootstrapRequest: request, + logger: logger, + once: sync.Once{}, + session: session.New(), + } +} + +// Close implements ProbeEngine +func (s *Session) Close() error { + s.once.Do(func() { + s.session.Close() + }) + return nil +} + +// BootstrapRequest returns the configured bootstrap request. +func (s *Session) BootstrapRequest() *session.BootstrapRequest { + return s.bootstrapRequest +} + +// Bootstrap performs the session bootstrap. +func (s *Session) Bootstrap(ctx context.Context) error { + if err := s.session.Send(ctx, &session.Request{Bootstrap: s.bootstrapRequest}); err != nil { + return err + } + for { + resp, err := s.session.Recv(ctx) + if err != nil { + return err + } + if resp.Log != nil { + s.emitLog(resp.Log) + continue + } + if resp.Bootstrap != nil { + return resp.Bootstrap.Error + } + } +} + +// Geolocate runs the geolocate task. +func (s *Session) Geolocate(ctx context.Context) (*geolocate.Results, error) { + req := &session.GeolocateRequest{} + if err := s.session.Send(ctx, &session.Request{Geolocate: req}); err != nil { + return nil, err + } + for { + resp, err := s.session.Recv(ctx) + if err != nil { + return nil, err + } + if resp.Log != nil { + s.emitLog(resp.Log) + continue + } + if resp.Geolocate != nil { + return resp.Geolocate.Location, resp.Geolocate.Error + } + } +} + +// CheckIn runs the checkIn task. +func (s *Session) CheckIn( + ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResult, error) { + if err := s.session.Send(ctx, &session.Request{CheckIn: config}); err != nil { + return nil, err + } + for { + resp, err := s.session.Recv(ctx) + if err != nil { + return nil, err + } + if resp.Log != nil { + s.emitLog(resp.Log) + continue + } + if resp.CheckIn != nil { + return resp.CheckIn.Result, resp.CheckIn.Error + } + } +} + +// Submit submits the given measurement. +func (s *Session) Submit(ctx context.Context, measurement *model.Measurement) error { + if err := s.session.Send(ctx, &session.Request{Submit: measurement}); err != nil { + return err + } + for { + resp, err := s.session.Recv(ctx) + if err != nil { + return err + } + if resp.Log != nil { + s.emitLog(resp.Log) + continue + } + if resp.Submit != nil { + return resp.Submit.Error + } + } +} + +// emitLog emits a log event. +func (s *Session) emitLog(ev *session.LogEvent) { + switch ev.Level { + case "DEBUG": + s.logger.Debug(ev.Message) + case "WARNING": + s.logger.Warn(ev.Message) + default: + s.logger.Info(ev.Message) + } +} diff --git a/internal/nettests/utils.go b/internal/nettests/utils.go new file mode 100644 index 0000000000..e0824859d4 --- /dev/null +++ b/internal/nettests/utils.go @@ -0,0 +1,17 @@ +package nettests + +import ( + "encoding/json" + "os" + + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// SaveMeasurement saves the given measurement in the given directory. +func SaveMeasurement(measurement *model.Measurement, filename string) error { + data, err := json.Marshal(measurement) + runtimex.PanicOnError(err, "json.Marshal failed") + data = append(data, byte('\n')) + return os.WriteFile(filename, data, 0600) +} diff --git a/internal/nettests/webconnectivity.go b/internal/nettests/webconnectivity.go new file mode 100644 index 0000000000..81d0130a5e --- /dev/null +++ b/internal/nettests/webconnectivity.go @@ -0,0 +1,141 @@ +package nettests + +import ( + "context" + "time" + + "github.com/ooni/probe-cli/v3/internal/bytecounter" + "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivity" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/session" +) + +// WebConnectivityFactoryConfig contains config for creating +// a new [WebConnectivityFactory] instance. +type WebConnectivityFactoryConfig struct { + // CheckIn contains the MANDATORY response returned by check-in. + CheckIn *model.OOAPICheckInResult + + // InputFiles contains the OPTIONAL input files to read. + InputFiles []string + + // Inputs contains the OPTIONAL inputs to read. + Inputs []string + + // Session is the MANDATORY session. + Session *Session +} + +// WebConnectivityFactory creates the [WebConnectivity] experiment. The zero +// value is invalid. Construct using [NewWebConnectivityFactory]. +type WebConnectivityFactory struct { + experiment *WebConnectivity +} + +// NewWebConnectivityFactory creates a new [WebConnectivity] factory. +func NewWebConnectivityFactory(config *WebConnectivityFactoryConfig) (*WebConnectivityFactory, error) { + runtimex.Assert(config != nil, "passed nil config") + runtimex.Assert(config.CheckIn != nil, "passed nil config.CheckIn") + runtimex.Assert(config.Session != nil, "passed nil config.Session") + if config.CheckIn.Tests.WebConnectivity == nil { + return nil, ErrMissingCheckInConfig + } + f := &WebConnectivityFactory{ + experiment: &WebConnectivity{ + byteCounter: bytecounter.New(), + config: config, + callbacks: model.NewPrinterCallbacks(model.DiscardLogger), + testStartTime: time.Now(), + }, + } + return f, nil +} + +var _ ExperimentFactory = &WebConnectivityFactory{} + +// LoadInputs loads the proper inputs for this experiment. +func (f *WebConnectivityFactory) LoadInputs() ([]model.OOAPIURLInfo, error) { + return loadInputs( + f.experiment.config.CheckIn.Tests.WebConnectivity.URLs, + f.experiment.config.InputFiles, + f.experiment.config.Inputs, + ) +} + +// NewExperiment implements ExperimentFactory +func (f *WebConnectivityFactory) NewExperiment(callbacks model.ExperimentCallbacks) Experiment { + f.experiment.callbacks = callbacks + return f.experiment +} + +// WebConnectivity is the Web Connectivity experiment. The zero value +// is invalid. Construct using [WebConnectivityFactory]. +type WebConnectivity struct { + // byteCounter is the byte counter we use. + byteCounter *bytecounter.Counter + + // config is the config with which we were created. + config *WebConnectivityFactoryConfig + + // callbacks contains the experiment callbacks. + callbacks model.ExperimentCallbacks + + // testStartTime is when we started this test. + testStartTime time.Time +} + +var _ Experiment = &WebConnectivity{} + +// GetSummaryKeys implements Experiment +func (e *WebConnectivity) GetSummaryKeys(m *model.Measurement) (any, error) { + return webconnectivity.GetSummaryKeys(m) +} + +// KibiBytesReceived implements Experiment +func (e *WebConnectivity) KibiBytesReceived() float64 { + return e.byteCounter.KibiBytesReceived() +} + +// KibiBytesSent implements Experiment +func (e *WebConnectivity) KibiBytesSent() float64 { + return e.byteCounter.KibiBytesSent() +} + +// Measure implements Experiment +func (e *WebConnectivity) Measure(ctx context.Context, input string) (*model.Measurement, error) { + // TODO(bassosimone): how to measure the bytes sent and received? + req := &session.WebConnectivityRequest{ + Input: input, + ReportID: e.config.CheckIn.Tests.WebConnectivity.ReportID, + TestStartTime: e.testStartTime, + } + runtimex.Assert(e.config.Session.session != nil, "expected non-nil Session.session") + sess := e.config.Session + if err := sess.session.Send(ctx, &session.Request{WebConnectivity: req}); err != nil { + return nil, err + } + for { + resp, err := sess.session.Recv(ctx) + if err != nil { + return nil, err + } + if resp.Log != nil { + sess.emitLog(resp.Log) + continue + } + if resp.WebConnectivity != nil { + return resp.WebConnectivity.Measurement, resp.WebConnectivity.Error + } + } +} + +// Name implements Experiment +func (e *WebConnectivity) Name() string { + return webconnectivity.ExperimentName +} + +// ReportID implements Experiment +func (e *WebConnectivity) ReportID() string { + return e.config.CheckIn.Tests.WebConnectivity.ReportID +} diff --git a/internal/netxlite/maybeproxy.go b/internal/netxlite/maybeproxy.go index edab70b19b..ceea868f76 100644 --- a/internal/netxlite/maybeproxy.go +++ b/internal/netxlite/maybeproxy.go @@ -7,6 +7,7 @@ package netxlite import ( "context" "errors" + "log" "net" "net/url" @@ -44,6 +45,7 @@ var ErrProxyUnsupportedScheme = errors.New("proxy: unsupported scheme") // DialContext implements Dialer.DialContext. func (d *proxyDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + log.Printf("HEY dialing with %s", d.ProxyURL) url := d.ProxyURL if url.Scheme != "socks5" { return nil, ErrProxyUnsupportedScheme diff --git a/internal/ooapi/submit.go b/internal/ooapi/submit.go new file mode 100644 index 0000000000..f6327dd69f --- /dev/null +++ b/internal/ooapi/submit.go @@ -0,0 +1,36 @@ +package ooapi + +import ( + "encoding/json" + "fmt" + "net/http" + + "github.com/ooni/probe-cli/v3/internal/httpapi" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// NewSubmitMeasurementDescriptor creates a new [httpapi.Descriptor] describing how +// to submit a measurement to the OONI backend. +func NewSubmitMeasurementDescriptor( + req *model.OOAPICollectorUpdateRequest, reportID string) *httpapi.Descriptor[ + *model.OOAPICollectorUpdateRequest, *model.OOAPICollectorUpdateResponse] { + rawBody, err := json.Marshal(req) + runtimex.PanicOnError(err, "json.Marshal failed") + return &httpapi.Descriptor[*model.OOAPICollectorUpdateRequest, *model.OOAPICollectorUpdateResponse]{ + Accept: httpapi.ApplicationJSON, + Authorization: "", + AcceptEncodingGzip: false, + ContentType: httpapi.ApplicationJSON, + LogBody: true, + MaxBodySize: 0, + Method: http.MethodPost, + Request: &httpapi.RequestDescriptor[*model.OOAPICollectorUpdateRequest]{ + Body: rawBody, + }, + Response: &httpapi.JSONResponseDescriptor[model.OOAPICollectorUpdateResponse]{}, + Timeout: 0, + URLPath: fmt.Sprintf("/report/%s", reportID), + URLQuery: nil, + } +} diff --git a/internal/optional/optional.go b/internal/optional/optional.go new file mode 100644 index 0000000000..7fcf4b0389 --- /dev/null +++ b/internal/optional/optional.go @@ -0,0 +1,101 @@ +// Package optional contains safer code to handle optional values. +package optional + +import ( + "bytes" + "encoding/json" + "reflect" + + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// Value is an optional value. The zero value of this structure +// is equivalent to the one you get when calling [None]. +type Value[T any] struct { + // indirect is the indirect pointer to the value. + indirect *T +} + +// None constructs an empty value. +func None[T any]() Value[T] { + return Value[T]{nil} +} + +// Some constructs a some value unless T is a pointer and points to +// nil, in which case [Some] is equivalent to [None]. +func Some[T any](value T) Value[T] { + v := Value[T]{} + maybeSetFromValue(&v, value) + return v +} + +// maybeSetFromValue sets the underlying value unless T is a pointer +// in which case we set the Value to be empty. +func maybeSetFromValue[T any](v *Value[T], value T) { + rv := reflect.ValueOf(value) + if rv.Type().Kind() == reflect.Pointer && rv.IsZero() { + v.indirect = nil + return + } + v.indirect = &value +} + +var _ json.Unmarshaler = &Value[int]{} + +// UnmarshalJSON implements json.Unmarshaler. Note that a `null` JSON +// value always leads to an empty Value. +func (v *Value[T]) UnmarshalJSON(data []byte) error { + // A `null` underlying value should always be equivalent to + // invoking the None constructor of for T. While this is not + // what the [json] package recommends doing for this case, + // it is consistent with initializing an optional. + if bytes.Equal(data, []byte(`null`)) { + v.indirect = nil + return nil + } + + // Otherwise, let's try to unmarshal into a real value + var value T + if err := json.Unmarshal(data, &value); err != nil { + return err + } + + // Enforce the same semantics of the Some constructor: treat + // pointer types specially to avoid the case where we have + // a Value that is wrapping a nil pointer but for which the + // IsNone check actually returns false. (Maybe this check is + // redundant but it seems better to enforce it anyway.) + maybeSetFromValue(v, value) + return nil +} + +var _ json.Marshaler = Value[int]{} + +// MarshalJSON implements json.Marshaler. An empty value serializes +// to `null` and otherwise we serialize the underluing value. +func (v Value[T]) MarshalJSON() ([]byte, error) { + if v.indirect == nil { + return json.Marshal(nil) + } + return json.Marshal(*v.indirect) +} + +// IsNone returns whether this [Value] is empty. +func (v Value[T]) IsNone() bool { + return v.indirect == nil +} + +// Unwrap returns the underlying value or panics. In case of +// panic, the value passed to panic is an error. +func (v Value[T]) Unwrap() T { + runtimex.Assert(!v.IsNone(), "is none") + return *v.indirect +} + +// UnwrapOr returns the fallback if the [Value] is empty. +func (v Value[T]) UnwrapOr(fallback T) T { + if v.IsNone() { + return fallback + } + return v.Unwrap() +} diff --git a/internal/optional/optional_test.go b/internal/optional/optional_test.go new file mode 100644 index 0000000000..dff22a0b0a --- /dev/null +++ b/internal/optional/optional_test.go @@ -0,0 +1,265 @@ +package optional + +import ( + "encoding/json" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestValue(t *testing.T) { + + // Verify that None creates a Value with an indirect == nil + t.Run("None works as intended", func(t *testing.T) { + v := None[int]() + if v.indirect != nil { + t.Fatal("should be nil") + } + }) + + t.Run("Some works as intended", func(t *testing.T) { + + // Verify that Some(value) creates a valid underlying pointer to + // the value when the wrapped type is not a pointer. + t.Run("for nonzero nonpointer value", func(t *testing.T) { + underlying := 12345 + v := Some(underlying) + if v.indirect == nil || *v.indirect != underlying { + t.Fatal("unexpected indirect") + } + }) + + // Verify that Some(value) works for a zero input when the + // wrapped value is not a pointer. + t.Run("for zero nonpointer value", func(t *testing.T) { + underlying := 0 + v := Some(underlying) + if v.indirect == nil || *v.indirect != underlying { + t.Fatal("unexpected indirect") + } + }) + + // Verify that Some(value) correctly creates a pointer to the + // underlying value when we're wrapping a pointer type + t.Run("for nonzero pointer value", func(t *testing.T) { + underlying := 12345 + v := Some(&underlying) + if v.indirect == nil || *v.indirect == nil || **v.indirect != underlying { + t.Fatal("unexpected indirect") + } + }) + + // Verify that Some(nil) creates an empty value when wrapping a pointer + t.Run("for zero nonpointer value", func(t *testing.T) { + var underlying *int + v := Some(underlying) + if v.indirect != nil { + t.Fatal("unexpected indirect", *v.indirect) + } + }) + }) + + t.Run("UnmarshalJSON works as intended", func(t *testing.T) { + + t.Run("for nonpointer type", func(t *testing.T) { + + // When we wrap a nonpointer and the JSON is valid, we expect + // the underlying value to be correctly populated + t.Run("with valid JSON input", func(t *testing.T) { + type config struct { + UID Value[int64] + } + + input := []byte(`{"UID":12345}`) + var state config + if err := json.Unmarshal(input, &state); err != nil { + t.Fatal(err) + } + + if state.UID.indirect == nil || *state.UID.indirect != 12345 { + t.Fatal("did not set indirect correctly") + } + }) + + // When the JSON input is incompatible, there should always + // be an error indicating we cannot assign and obviously the + // Value should not have been set. + t.Run("with incompatible JSON input", func(t *testing.T) { + type config struct { + UID Value[int64] + } + + input := []byte(`{"UID":[]}`) + var state config + err := json.Unmarshal(input, &state) + if err == nil || err.Error() != "json: cannot unmarshal array into Go struct field config.UID of type int64" { + t.Fatal("unexpected err", err) + } + + if state.UID.indirect != nil { + t.Fatal("should not have set", *state.UID.indirect) + } + }) + + // As a special case, when the JSON input is `null`, we should behave + // like the None constructor had been called. + t.Run("with null JSON input", func(t *testing.T) { + type config struct { + UID Value[int64] + } + + input := []byte(`{"UID":null}`) + var state config + err := json.Unmarshal(input, &state) + if err != nil { + t.Fatal(err) + } + + if state.UID.indirect != nil { + t.Fatal("should not have set", *state.UID.indirect) + } + }) + }) + + t.Run("for pointer type", func(t *testing.T) { + + // When the JSON input is valid, we expect that the underlying pointer + // is a pointer to the expected value. + t.Run("with valid JSON input", func(t *testing.T) { + type config struct { + UID Value[*int64] + } + + input := []byte(`{"UID":12345}`) + var state config + if err := json.Unmarshal(input, &state); err != nil { + t.Fatal(err) + } + + if state.UID.indirect == nil || *state.UID.indirect == nil || **state.UID.indirect != 12345 { + t.Fatal("did not set indirect correctly") + } + }) + + // With incompatible JSON input, there should be an error and obviously + // we should not have set any value inside the Value + t.Run("with incompatible JSON input", func(t *testing.T) { + type config struct { + UID Value[*int64] + } + + input := []byte(`{"UID":[]}`) + var state config + err := json.Unmarshal(input, &state) + if err == nil || err.Error() != "json: cannot unmarshal array into Go struct field config.UID of type int64" { + t.Fatal("unexpected err", err) + } + + if state.UID.indirect != nil { + t.Fatal("should not have set", *state.UID.indirect) + } + }) + + // When the JSON input is `null`, the code should behave like we + // had invoked the None constructor for the pointer type. + t.Run("with null JSON input", func(t *testing.T) { + type config struct { + UID Value[*int64] + } + + input := []byte(`{"UID":null}`) + var state config + err := json.Unmarshal(input, &state) + if err != nil { + t.Fatal(err) + } + + if state.UID.indirect != nil { + t.Fatal("should not have set", *state.UID.indirect) + } + }) + }) + }) + + t.Run("MarshalJSON works as intended", func(t *testing.T) { + t.Run("for an empty Value", func(t *testing.T) { + value := None[int]() + got, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + expect := []byte(`null`) + if diff := cmp.Diff(expect, got); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("for an nonempty Value", func(t *testing.T) { + value := Some(12345) + got, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + expect := []byte(`12345`) + if diff := cmp.Diff(expect, got); diff != "" { + t.Fatal(diff) + } + }) + }) + + t.Run("IsNone works as intended", func(t *testing.T) { + t.Run("for empty Value", func(t *testing.T) { + value := None[int]() + if !value.IsNone() { + t.Fatal("should be none") + } + }) + + t.Run("for nonempty Value", func(t *testing.T) { + value := Some(12345) + if value.IsNone() { + t.Fatal("should not be none") + } + }) + }) + + t.Run("Unwrap works as intended", func(t *testing.T) { + t.Run("for an empty Value", func(t *testing.T) { + value := None[int]() + var err error + func() { + defer func() { + err = recover().(error) + }() + out := value.Unwrap() + t.Log(out) + }() + if err == nil || err.Error() != "is none" { + t.Fatal("unexpected err", err) + } + }) + + t.Run("for a nonempty Value", func(t *testing.T) { + value := Some(12345) + if v := value.Unwrap(); v != 12345 { + t.Fatal("unexpected value", v) + } + }) + }) + + t.Run("UnwrapOr works as intended", func(t *testing.T) { + t.Run("for an empty Value", func(t *testing.T) { + value := None[int]() + if v := value.UnwrapOr(555); v != 555 { + t.Fatal("unexpected value", v) + } + }) + + t.Run("for a nonempty Value", func(t *testing.T) { + value := Some(12345) + if v := value.UnwrapOr(555); v != 12345 { + t.Fatal("unexpected value", v) + } + }) + }) +} diff --git a/internal/session/bootstrap.go b/internal/session/bootstrap.go new file mode 100644 index 0000000000..7581bba084 --- /dev/null +++ b/internal/session/bootstrap.go @@ -0,0 +1,98 @@ +package session + +// +// Bootstrapping a measurement session. +// + +import ( + "context" + + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// BootstrapRequest is a request to bootstrap the [Session] and +// contains the arguments required by the bootstrap. You can +// boostrap a [Session] just once. All operations you would like +// to perform with a [Session] require a boostrap first. +type BootstrapRequest struct { + // SnowflakeRendezvousMethod is the OPTIONAL rendezvous + // method to use when ProxyURL scheme is `torsf`. + SnowflakeRendezvousMethod string + + // StateDir is the MANDATORY directory where to store + // persistent engine state using a key-value store. + StateDir string + + // ProxyURL is the OPTIONAL proxy URL. We accept the + // following proxy URL schemes: + // + // - "socks5": configures a socks5 proxy; + // + // - "tor": requires a tor tunnel; + // + // - "torsf": requires a tor+snowflake tunnel; + // + // - "psiphon": requires a psiphon tunnel. + // + // When requesting a tunnel, we only check the URL scheme + // and disregard the rest of the URL. + ProxyURL string + + // SoftwareName is the MANDATORY software name. + SoftwareName string + + // SoftwareVersion is the MANDATORY software version. + SoftwareVersion string + + // TorArgs OPTIONALLY passes command line arguments to tor + // when the ProxyURL scheme is "tor" or "torsf". We will only + // use these arguments for bootstrapping, not for measuring. + TorArgs []string + + // TorBinary OPTIONALLY tells the engine to use a specific + // binary for starting the "tor" and "torsf" tunnels. If this + // argument is set, we will also use it for measuring for + // each experiment that requires tor. + TorBinary string + + // TempDir is the MANDATORY base directory in which + // the session should store temporary state. + TempDir string + + // TunnelDir is the MANDATORY directory in which + // to store persistent tunnel state. + TunnelDir string + + // VerboseLogging OPTIONALLY enables verbose logging. + VerboseLogging bool +} + +// BootstrapEvent is the event emmitted at the end of the bootstrap. +type BootstrapEvent struct { + // Error is the bootstrap result. + Error error +} + +// boostrap bootstraps a session. +func (s *Session) bootstrap(ctx context.Context, req *BootstrapRequest) { + runtimex.Assert(req != nil, "passed nil req") + s.maybeEmit(&Event{ + Bootstrap: &BootstrapEvent{ + Error: s.dobootstrap(ctx, req), + }, + }) +} + +// dobootstrap implements bootstrap. +func (s *Session) dobootstrap(ctx context.Context, req *BootstrapRequest) error { + if s.state.IsSome() { + return nil // idempotent + } + state, err := s.newState(ctx, req) + if err != nil { + return err + } + s.state = model.NewOptionalPtr(state) + return nil +} diff --git a/internal/session/checkin.go b/internal/session/checkin.go new file mode 100644 index 0000000000..791b892688 --- /dev/null +++ b/internal/session/checkin.go @@ -0,0 +1,59 @@ +package session + +// +// Code to call the check-in API. +// + +import ( + "context" + + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// CheckInRequest is the request for the check-in API. +type CheckInRequest = model.OOAPICheckInConfig + +// CheckInEvent is the result of calling the check-in API. +type CheckInEvent struct { + // Error is the geolocate result. + Error error + + // Result is the result returned on success. + Result *model.OOAPICheckInResult +} + +// checkin calls the check-in API. +func (s *Session) checkin(ctx context.Context, req *CheckInRequest) { + runtimex.Assert(req != nil, "passed a nil req") + result, err := s.docheckin(ctx, req) + s.maybeEmit(&Event{ + CheckIn: &CheckInEvent{ + Error: err, + Result: result, + }, + }) +} + +// docheckin implements checkin. +func (s *Session) docheckin(ctx context.Context, req *CheckInRequest) (*model.OOAPICheckInResult, error) { + if s.state.IsNone() { + return nil, ErrNotBootstrapped + } + if s.state.Unwrap().checkIn.IsSome() { + // TODO(bassosimone): in the future we should define caching + // policies for the check-in response, but for now this is fine. + return s.state.Unwrap().checkIn.Unwrap(), nil + } + + ts := newTickerService(ctx, s) + defer ts.stop() + + backendClient := s.state.Unwrap().backendClient + result, err := backendClient.CheckIn(ctx, req) + if err != nil { + return nil, err + } + s.state.Unwrap().checkIn = model.NewOptionalPtr(result) + return result, nil +} diff --git a/internal/session/doc.go b/internal/session/doc.go new file mode 100644 index 0000000000..9f37164713 --- /dev/null +++ b/internal/session/doc.go @@ -0,0 +1,5 @@ +// Package session implements a measurement session. The design of +// this package is such that we can split the measurement engine proper +// and the application using it. In particular, this design is such +// that it would be easy to expose this API as a C library. +package session diff --git a/internal/session/geolocate.go b/internal/session/geolocate.go new file mode 100644 index 0000000000..9a0d70a710 --- /dev/null +++ b/internal/session/geolocate.go @@ -0,0 +1,74 @@ +package session + +// +// Geolocating a probe. +// + +import ( + "context" + "errors" + + "github.com/ooni/probe-cli/v3/internal/geolocate" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// GeolocateRequest contains config for geolocate. +type GeolocateRequest struct{} + +// GeolocateEvent is the event emitted at the end of geolocate. +type GeolocateEvent struct { + // Error is the geolocate result. + Error error + + // Location is the geolocated location. + Location *geolocate.Results +} + +// geolocate performs a geolocation. +func (s *Session) geolocate(ctx context.Context, req *GeolocateRequest) { + runtimex.Assert(req != nil, "passed a nil req") + location, err := s.dogeolocate(ctx, req) + s.maybeEmit(&Event{ + Geolocate: &GeolocateEvent{ + Error: err, + Location: location, + }, + }) +} + +// ErrNotBootstrapped indicates we have not bootstrapped the session yet. +var ErrNotBootstrapped = errors.New("session: not bootstrapped") + +// dogeolocate implements geolocate. +func (s *Session) dogeolocate(ctx context.Context, req *GeolocateRequest) (*geolocate.Results, error) { + if s.state.IsNone() { + return nil, ErrNotBootstrapped + } + if s.state.Unwrap().location.IsSome() { + // TODO(bassosimone): in the future we should define caching + // policies for the location, but for now this is fine. + return s.state.Unwrap().location.Unwrap(), nil + } + + ts := newTickerService(ctx, s) + defer ts.stop() + + geolocateConfig := geolocate.Config{ + Resolver: s.state.Unwrap().resolver, + Logger: s.state.Unwrap().logger, + UserAgent: model.HTTPHeaderUserAgent, // do not disclose we are OONI + } + task := geolocate.NewTask(geolocateConfig) // TODO(bassosimone): make this a pointer. + + // TODO(bassosimone): we should make geolocate.Results a type + // in the internal/model package and use the ~typical naming for + // its fields rather than the naming we have here. + location, err := task.Run(ctx) + if err != nil { + return nil, err + } + + s.state.Unwrap().location = model.NewOptionalPtr(location) + return location, nil +} diff --git a/internal/session/logger.go b/internal/session/logger.go new file mode 100644 index 0000000000..067dbcfe0c --- /dev/null +++ b/internal/session/logger.go @@ -0,0 +1,85 @@ +package session + +// +// A model.Logger emitting events on a Session output channel +// + +import ( + "fmt" + "time" + + "github.com/ooni/probe-cli/v3/internal/model" +) + +// LogEvent is a log event. +type LogEvent struct { + // Timestamp is the log timestamp. + Timestamp time.Time + + // Level is the log level. + Level string + + // Message is the log message. + Message string +} + +// newLogger creates a [model.Logger] that emits +// [LogEvent] events using the [Session]. +func (s *Session) newLogger(verbose bool) model.Logger { + return &sessionLogger{ + session: s, + verbose: verbose, + } +} + +// sessionLogger is a [model.Logger] using a [Session]. +type sessionLogger struct { + session *Session + verbose bool +} + +// Debug implements model.Logger +func (sl *sessionLogger) Debug(msg string) { + if sl.verbose { + sl.maybeEmit("DEBUG", msg) + } +} + +// Debugf implements model.Logger +func (sl *sessionLogger) Debugf(format string, v ...interface{}) { + if sl.verbose { + sl.Debug(fmt.Sprintf(format, v...)) + } +} + +// Info implements model.Logger +func (sl *sessionLogger) Info(msg string) { + sl.maybeEmit("INFO", msg) +} + +// Infof implements model.Logger +func (sl *sessionLogger) Infof(format string, v ...interface{}) { + sl.Info(fmt.Sprintf(format, v...)) +} + +// Warn implements model.Logger +func (sl *sessionLogger) Warn(msg string) { + sl.maybeEmit("WARNING", msg) +} + +// Warnf implements model.Logger +func (sl *sessionLogger) Warnf(format string, v ...interface{}) { + sl.Warn(fmt.Sprintf(format, v...)) +} + +// maybeEmit emits a log message if the output channel buffer is not full. +func (sl *sessionLogger) maybeEmit(level, message string) { + ev := &Event{ + Log: &LogEvent{ + Timestamp: time.Now(), + Level: level, + Message: message, + }, + } + sl.session.maybeEmit(ev) +} diff --git a/internal/session/progressbar.go b/internal/session/progressbar.go new file mode 100644 index 0000000000..7928c484b2 --- /dev/null +++ b/internal/session/progressbar.go @@ -0,0 +1,50 @@ +package session + +// +// Progress bar for experiments that manage their own progress +// bar such as DASH, NDT, HIRL, HHFM. +// + +import ( + "time" + + "github.com/ooni/probe-cli/v3/internal/model" +) + +// ProgressEvent indicates the progress in a task completion +// for tasks that manage their own progress bar. +type ProgressEvent struct { + // Timestamp is the timestamp. + Timestamp time.Time + + // Completion is a number between 0 and 1 indicating + // how close we are to completion. + Completion float64 + + // Message is the message. + Message string +} + +// progressBar emits progress. +type progressBar struct { + session *Session +} + +var _ model.ExperimentCallbacks = &progressBar{} + +// OnProgress implements model.ExperimentCallbacks +func (pb *progressBar) OnProgress(completion float64, message string) { + pb.maybeEmit(completion, message) +} + +// emit emits a progress event unless the output channel is full. +func (pb *progressBar) maybeEmit(completion float64, message string) { + ev := &Event{ + Progress: &ProgressEvent{ + Timestamp: time.Now(), + Completion: completion, + Message: message, + }, + } + pb.session.maybeEmit(ev) +} diff --git a/internal/session/session.go b/internal/session/session.go new file mode 100644 index 0000000000..8dccce4868 --- /dev/null +++ b/internal/session/session.go @@ -0,0 +1,227 @@ +package session + +// +// Public definition of Session +// + +import ( + "context" + "errors" + "log" + "sync" + + "github.com/ooni/probe-cli/v3/internal/model" +) + +// Session is a measurement session. The zero value of this structure +// is invalid. You must use the [New] factory to create a new valid instance. +// +// A session consists of a background goroutine to which you Send +// [Request]s. Each [Request] causes the background goroutine to +// start running a long-running task. You should Recv [Event]s +// emitted by the task until you find the matching [Event] that implies +// that the task you started has finished running. +// +// While running, a task emits "ticker" events that you can use to +// fill a progress bar and to decide when the task should stop running. To +// stop a long-running task, you call [Close], which forces the background +// goroutine to stop as soon as possible. +// +// Once a [Session] has been terminated using [Close] you loose all +// the [Session] state and you must create a new [Session]. +type Session struct { + // cancel allows us to terminate the backround goroutine. + cancel context.CancelFunc + + // input is the channel to send [Request] to the background goroutine. + input chan *Request + + // once allows us to cleanup the state just once. + once sync.Once + + // output is the channel from which we read the emitted [Event]s. + output chan *Event + + // state is the background goroutine's state, which only + // the background goroutine is allowed to modify. We start + // with a nil state and create state using the bootstrap + // long running task. + state model.OptionalPtr[state] + + // terminated is closed when the background goroutine terminates. + terminated chan any +} + +// New creates a new measurement [Session]. This function will create +// a background goroutine that will handle incoming [Request]s. +func New() *Session { + ctx, cancel := context.WithCancel(context.Background()) + // Implementation note: we use buffered channels for input and + // output to avoid loosing events in _most_ cases. + s := &Session{ + cancel: cancel, + input: make(chan *Request, 1024), + once: sync.Once{}, + output: make(chan *Event, 1024), + state: model.OptionalPtr[state]{}, + terminated: make(chan any), + } + go s.mainloop(ctx) + return s +} + +// Request requests a [Session] to perform a background task. This +// struct contains several pointers. Each of them indicates a specific +// task the [Session] should run. The [Session] will go through each +// pointer and only consider the first one that is not nil. As such +// it's pointless to initialize more than one pointer. +type Request struct { + // Bootstrap indicates that the [Session] should bootstrap + // and contains bootstrap configuration. + Bootstrap *BootstrapRequest + + // CheckIn indicates that the [Session] should call the check-in API + // and only works if you have already bootstrapped. + CheckIn *CheckInRequest + + // Geolocate indicates that the [Session] should obtain + // the current probe's geolocation. This task requires you + // to successfully bootstrap a session first. + Geolocate *GeolocateRequest + + // Submit indicates that the [Session] should submit a + // measurement. You must bootstrap first. + Submit *SubmitRequest + + // WebConnectivity indicates that the [Session] should + // run the Web Connectivity experiment. You must bootstrap first. + WebConnectivity *WebConnectivityRequest +} + +// ErrSessionTerminated indicates that the background goroutine has terminated. +var ErrSessionTerminated = errors.New("session: terminated") + +// Send sends a [Request] to a [Session]. This function will return an +// error when the [Session] has been stopped or when the context has expired. +func (s *Session) Send(ctx context.Context, req *Request) error { + select { + case s.input <- req: + return nil + case <-s.terminated: + return ErrSessionTerminated + case <-ctx.Done(): + return ctx.Err() + } +} + +// Close stops the goroutine running in the background and releases +// all the resources allocated by the [Session]. +func (s *Session) Close() error { + s.once.Do(s.joincleanup) + return nil +} + +// joincleanup joins the [Session] and closes open resources. +func (s *Session) joincleanup() { + s.cancel() + <-s.terminated + if s.state.IsSome() { + s.state.Unwrap().cleanup() + s.state = model.OptionalPtr[state]{} // just to be tidy + } +} + +// Recv receives the next [Event] emitted by a [Session]. This function +// will return an error when the [Session] has been stopped or when +// the context has expired. +func (s *Session) Recv(ctx context.Context) (*Event, error) { + select { + case ev := <-s.output: + return ev, nil + case <-s.terminated: + return nil, ErrSessionTerminated + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +// maybeEmit emits an [Event] if possible. If the output channel +// buffer is full, we are not going to emit the event. In such +// a case, we will print a log message to the standard error file. +func (s *Session) maybeEmit(ev *Event) { + select { + case s.output <- ev: + default: + log.Printf("session: cannot send event: %+v", ev) + } +} + +// Event is an event emitted by a [Session]. Only one of the pointers +// of the [Event] will be set. The pointer being set uniquely identifies +// the specific event that has occurred. +type Event struct { + // Bootstrap is emitted at the end of the bootstrap. + Bootstrap *BootstrapEvent + + // CheckIn is the event emitted at the end of the check-in. + CheckIn *CheckInEvent + + // Geolocate is emitted at the end of geolocate. + Geolocate *GeolocateEvent + + // Log is a log event. Any task will emit log events. + Log *LogEvent + + // Progress is a progress event. Only experiments that print + // their own progress will emit this event. + Progress *ProgressEvent + + // Submit is emitted after a measurement submission. + Submit *SubmitEvent + + // Ticker is a ticker event. Any task will emit ticker + // events so that you can increase a progress bar and + // decide whether the task should be stopped. + Ticker *TickerEvent + + // WebConnectivity is the Web Connectivity event, emitted + // once we finished measuring a given URL. + WebConnectivity *WebConnectivityEvent +} + +// mainloop runs the [Session] main loop. +func (s *Session) mainloop(ctx context.Context) { + defer close(s.terminated) + for { + select { + case <-ctx.Done(): + return + case req := <-s.input: + s.handle(ctx, req) + } + } +} + +// handle handles an incoming [Request]. +func (s *Session) handle(ctx context.Context, req *Request) { + if req.Bootstrap != nil { + s.bootstrap(ctx, req.Bootstrap) + return + } + if req.CheckIn != nil { + s.checkin(ctx, req.CheckIn) + return + } + if req.Geolocate != nil { + s.geolocate(ctx, req.Geolocate) + return + } + if req.Submit != nil { + s.submit(ctx, req.Submit) + return + } + if req.WebConnectivity != nil { + s.webconnectivity(ctx, req.WebConnectivity) + return + } +} diff --git a/internal/session/sessionadapter.go b/internal/session/sessionadapter.go new file mode 100644 index 0000000000..58eccea016 --- /dev/null +++ b/internal/session/sessionadapter.go @@ -0,0 +1,117 @@ +package session + +// +// Adapter to pass model.ExperimentSession to experiments +// + +import ( + "context" + "errors" + + "github.com/ooni/probe-cli/v3/internal/geolocate" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// sessionAdapter adapts [Session] to be a [model.ExperimentSesion]. +type sessionAdapter struct { + httpClient model.HTTPClient + location *geolocate.Results + logger model.Logger + tempDir string + testHelpers map[string][]model.OOAPIService + torBinary string + tunnelDir string + userAgent string +} + +// ErrNoLocation means we cannot proceed without knowing the probe location. +var ErrNoLocation = errors.New("session: no location information") + +// ErrNoCheckIn means we cannot proceed without the check-in API results. +var ErrNoCheckIn = errors.New("session: no check-in information") + +// newSessionAdapter creates a new [sessionAdapter] instance. +func newSessionAdapter(state *state) (*sessionAdapter, error) { + if state.location.IsNone() { + return nil, ErrNoLocation + } + if state.checkIn.IsNone() { + return nil, ErrNoCheckIn + } + sa := &sessionAdapter{ + httpClient: state.httpClient, + location: state.location.Unwrap(), + logger: state.logger, + tempDir: state.tempDir, + testHelpers: state.checkIn.Unwrap().Conf.TestHelpers, + torBinary: state.torBinary, + tunnelDir: state.tunnelDir, + userAgent: state.userAgent, + } + return sa, nil +} + +var _ model.ExperimentSession = &sessionAdapter{} + +// DefaultHTTPClient implements model.ExperimentSession +func (es *sessionAdapter) DefaultHTTPClient() model.HTTPClient { + return es.httpClient +} + +// FetchPsiphonConfig implements model.ExperimentSession +func (es *sessionAdapter) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { + return nil, errors.New("not implemented") +} + +// FetchTorTargets implements model.ExperimentSession +func (es *sessionAdapter) FetchTorTargets(ctx context.Context, cc string) (map[string]model.OOAPITorTarget, error) { + return nil, errors.New("not implemented") +} + +// GetTestHelpersByName implements model.ExperimentSession +func (es *sessionAdapter) GetTestHelpersByName(name string) ([]model.OOAPIService, bool) { + value, found := es.testHelpers[name] + return value, found +} + +// Logger implements model.ExperimentSession +func (es *sessionAdapter) Logger() model.Logger { + return es.logger +} + +// ProbeCC implements model.ExperimentSession +func (es *sessionAdapter) ProbeCC() string { + return es.location.CountryCode +} + +// ResolverIP implements model.ExperimentSession +func (es *sessionAdapter) ResolverIP() string { + return es.location.ResolverIPAddr +} + +// TempDir implements model.ExperimentSession +func (es *sessionAdapter) TempDir() string { + return es.tempDir +} + +// TorArgs implements model.ExperimentSession +func (es *sessionAdapter) TorArgs() []string { + // TODO(bassosimone): this field is only meaningful for bootstrap. So, it is + // wrong that we include it into the ExperimentSession. + return []string{} +} + +// TorBinary implements model.ExperimentSession +func (es *sessionAdapter) TorBinary() string { + return es.torBinary +} + +// TunnelDir implements model.ExperimentSession +func (es *sessionAdapter) TunnelDir() string { + return es.tunnelDir +} + +// UserAgent implements model.ExperimentSession +func (es *sessionAdapter) UserAgent() string { + return es.userAgent +} diff --git a/internal/session/state.go b/internal/session/state.go new file mode 100644 index 0000000000..b1353c795c --- /dev/null +++ b/internal/session/state.go @@ -0,0 +1,231 @@ +package session + +// +// State of a bootstrapped session. +// + +import ( + "context" + "errors" + "fmt" + "os" + + "github.com/ooni/probe-cli/v3/internal/backendclient" + "github.com/ooni/probe-cli/v3/internal/bytecounter" + "github.com/ooni/probe-cli/v3/internal/geolocate" + "github.com/ooni/probe-cli/v3/internal/kvstore" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/sessionhttpclient" + "github.com/ooni/probe-cli/v3/internal/sessionresolver" + "github.com/ooni/probe-cli/v3/internal/tunnel" + "github.com/ooni/probe-cli/v3/internal/version" +) + +// state is the bootstrapped [Session] state. Only the background +// goroutine is allowed to manipulate the [state]. +type state struct { + // backendClient is the backend client we're using. + backendClient *backendclient.Client + + // checkIn is the most recently fetched check-in result. + checkIn model.OptionalPtr[model.OOAPICheckInResult] + + // counter is the bytecounter we're using. + counter *bytecounter.Counter + + // httpClient is the HTTP client we're using. + httpClient model.HTTPClient + + // kvstore is the session's key-value store. + kvstore model.KeyValueStore + + // location is the most recently resolved location. + location model.OptionalPtr[geolocate.Results] + + // logger is the model.Logger we're using. + logger model.Logger + + // resolver is the session resolver. + resolver model.Resolver + + // softwareName is the software name to use. + softwareName string + + // softwareVersion is the software version to use. + softwareVersion string + + // tempDir is the session specific temporary directory. + tempDir string + + // torBinary contains the location of the tor binary to use. + torBinary string + + // torArgs should not be exposed here because we only + // want to use it for bootstrapping tor. + //torArgs []string + + // tunnelDir is the directory to use for tunnels. + tunnelDir string + + // tunnel is the tunnel we're using. + tunnel tunnel.Tunnel + + // userAgent is the user agent we're using when communicating + // with the OONI backend (e.g., for the test helpers). + userAgent string +} + +// cleanup cleans the resources used by [state]. +func (s *state) cleanup() { + s.resolver.CloseIdleConnections() + s.httpClient.CloseIdleConnections() + s.tunnel.Stop() + os.RemoveAll(s.tempDir) +} + +// ErrEmptySoftwareName indicates the software name is empty. +var ErrEmptySoftwareName = errors.New("session: passed empty software name") + +// ErrEmptySoftwareVersion indicates the software version is empty. +var ErrEmptySoftwareVersion = errors.New("session: passed empty software version") + +// newState creates a new [state] instance. +func (s *Session) newState(ctx context.Context, req *BootstrapRequest) (*state, error) { + if req.SoftwareName == "" { + return nil, ErrEmptySoftwareName + } + if req.SoftwareVersion == "" { + return nil, ErrEmptySoftwareVersion + } + + logger := s.newLogger(req.VerboseLogging) + + // Implementation note: the context we receive from the caller limits the + // whole lifetime of the tunnel we're going to create below. Because of + // that, we're not allowed to cancel this context or add a timeout to it. + + ts := newTickerService(ctx, s) + defer ts.stop() + + logger.Infof("bootstrap: creating key-value store at %s", req.StateDir) + kvstore, err := kvstore.NewFS(req.StateDir) + if err != nil { + logger.Warnf("bootstrap: cannot create key-value store: %s", err.Error()) + return nil, err + } + + logger.Infof("bootstrap: creating tunnels dir at %s", req.TunnelDir) + if err := os.MkdirAll(req.TunnelDir, 0700); err != nil { + logger.Warnf("bootstrap: cannot create tunnels dir: %s", err.Error()) + return nil, err + } + + tempDir, err := stateNewTempDir(logger, req) + if err != nil { + // warning message already printed + return nil, err + } + + tunnel, err := newTunnel(ctx, logger, req) + if err != nil { + logger.Warnf("bootstrap: cannot create tunnel: %s", err.Error()) + return nil, err + } + + state := newStateCannotFail( + logger, + kvstore, + tunnel, + tempDir, + req, + ) + return state, nil +} + +// newStateCannotFail constructs a [state] once we have +// performed all operations that may fail. +func newStateCannotFail( + logger model.Logger, + kvstore model.KeyValueStore, + tunnel tunnel.Tunnel, + tempDir string, + req *BootstrapRequest, +) *state { + runtimex.Assert(logger != nil, "passed a nil logger") + runtimex.Assert(kvstore != nil, "passed a nil kvstore") + runtimex.Assert(tunnel != nil, "passed a nil tunnel") + runtimex.Assert(tempDir != "", "passed an empty tempDir") + runtimex.Assert(req != nil, "passed a nil req") + + logger.Infof("bootstrap: creating a session byte counter") + counter := bytecounter.New() + + logger.Infof("bootstrap: creating a resolver for the session") + resolver := &sessionresolver.Resolver{ + ByteCounter: counter, + KVStore: kvstore, + Logger: logger, + ProxyURL: tunnel.SOCKS5ProxyURL(), // possibly nil, which is OK + } + + logger.Infof("bootstrap: creating an HTTP client for the session") + httpClient := sessionhttpclient.New(&sessionhttpclient.Config{ + ByteCounter: counter, + Logger: logger, + ProxyURL: tunnel.SOCKS5ProxyURL(), // possibly nil, which is OK + Resolver: resolver, + }) + + logger.Infof("bootstrap: creating the default user-agent string") + userAgent := fmt.Sprintf( + "%s/%s ooniprobe-engine/%s", + req.SoftwareName, + req.SoftwareVersion, + version.Version, + ) + + logger.Infof("bootstrap: creating an OONI backend client") + backendClient := backendclient.New(&backendclient.Config{ + BaseURL: nil, // use the default + KVStore: kvstore, + HTTPClient: httpClient, + Logger: logger, + UserAgent: userAgent, + }) + + logger.Infof("bootstrap: complete") + state := &state{ + backendClient: backendClient, + checkIn: model.OptionalPtr[model.OOAPICheckInResult]{}, + counter: counter, + httpClient: httpClient, + kvstore: kvstore, + location: model.OptionalPtr[geolocate.Results]{}, + logger: logger, + resolver: resolver, + softwareName: req.SoftwareName, + softwareVersion: req.SoftwareVersion, + tempDir: tempDir, + torBinary: req.TorBinary, + tunnelDir: req.TunnelDir, + tunnel: tunnel, + userAgent: userAgent, + } + return state +} + +// stateNewTempDir creates a new temporary directory for [state]. +func stateNewTempDir(logger model.Logger, req *BootstrapRequest) (string, error) { + logger.Infof("bootstrap: creating temporary directory inside %s", req.TempDir) + if err := os.MkdirAll(req.TempDir, 0700); err != nil { + logger.Warnf("bootstrap: cannot create temporary directory root: %s", err.Error()) + return "", err + } + tempDir, err := os.MkdirTemp(req.TempDir, "") + if err != nil { + logger.Warnf("bootstrap: cannot create session temporary directory: %s", err.Error()) + return "", err + } + return tempDir, nil +} diff --git a/internal/session/submit.go b/internal/session/submit.go new file mode 100644 index 0000000000..cb7bc2b84b --- /dev/null +++ b/internal/session/submit.go @@ -0,0 +1,44 @@ +package session + +// +// Submitting measurements +// + +import ( + "context" + + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// SubmitRequest is the request for the measurement submission API. +type SubmitRequest = model.Measurement + +// SubmitEvent is the result of submitting a measurement. +type SubmitEvent struct { + // Error is the geolocate result. + Error error +} + +// submit submits a measurement. +func (s *Session) submit(ctx context.Context, req *SubmitRequest) { + s.maybeEmit(&Event{ + Submit: &SubmitEvent{ + Error: s.dosubmit(ctx, req), + }, + }) +} + +// dosubmit implements submit. +func (s *Session) dosubmit(ctx context.Context, req *SubmitRequest) error { + runtimex.Assert(req != nil, "passed a nil req") + + if s.state.IsNone() { + return ErrNotBootstrapped + } + + ts := newTickerService(ctx, s) + defer ts.stop() + + return s.state.Unwrap().backendClient.Submit(ctx, req) +} diff --git a/internal/session/tickerservice.go b/internal/session/tickerservice.go new file mode 100644 index 0000000000..22b5e57327 --- /dev/null +++ b/internal/session/tickerservice.go @@ -0,0 +1,59 @@ +package session + +import ( + "context" + "time" +) + +// TickerEvent is an event emmitted by the [tickerService]. +type TickerEvent struct { + // ElapsedTime is the elapsed time since when the + // long running operation has been started. + ElapsedTime time.Duration +} + +// tickerServer is a ticker that emits a tick while a long running +// operation is alive. We use this ticker in the UI to make progress +// bars increase as well as to unblock the UI code and know when +// an operation has been running for too much time. +type tickerService struct { + cancel context.CancelFunc + sess *Session +} + +// newTickerService creates a [tickerService] running in the background +// and using the [Session] to emit [TickerEvents]. You should use the +// stop method of the [tickerService] to stop emitting events. +func newTickerService(ctx context.Context, sess *Session) *tickerService { + ctx, cancel := context.WithCancel(ctx) + ts := &tickerService{ + cancel: cancel, + sess: sess, + } + go ts.mainloop(ctx) + return ts +} + +// mainloop is the main loop of the [tickerService]. +func (ts *tickerService) mainloop(ctx context.Context) { + t0 := time.Now() + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case t := <-ticker.C: + ts.sess.maybeEmit(&Event{ + Ticker: &TickerEvent{ + ElapsedTime: t.Sub(t0), + }, + }) + case <-ctx.Done(): + return + } + } +} + +// stop stops the [tickerService] background goroutine. +func (ts *tickerService) stop() { + ts.cancel() +} diff --git a/internal/session/tunnel.go b/internal/session/tunnel.go new file mode 100644 index 0000000000..c27e582fa2 --- /dev/null +++ b/internal/session/tunnel.go @@ -0,0 +1,122 @@ +package session + +// +// Creating tunnels +// + +import ( + "context" + "errors" + "fmt" + "net/url" + "time" + + "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/tunnel" +) + +// ErrUnsupportedTunnelScheme indicates we don't support the tunnel scheme. +var ErrUnsupportedTunnelScheme = errors.New("session: unsupported tunnel scheme") + +// newTunnel creates a new [tunnel.Tunnel] given a ProxyURL. +func newTunnel(ctx context.Context, logger model.Logger, req *BootstrapRequest) (tunnel.Tunnel, error) { + if req.ProxyURL == "" { + logger.Info("bootstrap: no need to create any tunnel") + return &nullTunnel{}, nil + } + URL, err := url.Parse(req.ProxyURL) + if err != nil { + return nil, err + } + switch scheme := URL.Scheme; scheme { + case "socks5": + logger.Infof("bootstrap: creating fake tunnel for %s", req.ProxyURL) + return &socks5Tunnel{URL}, nil + case "tor", "torsf": + logger.Infof("bootstrap: creating %s tunnel; please, be patient...", scheme) + return newTorOrTorsfTunnel(ctx, logger, req, scheme) + case "psiphon": + logger.Info("bootstrap: creating psiphon tunnel; please, be patient...") + return newPsiphonTunnel(ctx, logger, req) + default: + return nil, fmt.Errorf("%w: %s", ErrUnsupportedTunnelScheme, scheme) + } +} + +// nullTunnel is the absence of any tunnel. +type nullTunnel struct{} + +// BootstrapTime implements tunnel.Tunnel +func (t *nullTunnel) BootstrapTime() time.Duration { + return 0 +} + +// SOCKS5ProxyURL implements tunnel.Tunnel +func (t *nullTunnel) SOCKS5ProxyURL() *url.URL { + return nil // perfectly fine to return nil in this context +} + +// Stop implements tunnel.Tunnel +func (t *nullTunnel) Stop() { + // nothing +} + +// socks5Tunnel is a fake tunnel that returns a SOCKS5 URL. +type socks5Tunnel struct { + url *url.URL +} + +// BootstrapTime implements tunnel.Tunnel +func (t *socks5Tunnel) BootstrapTime() time.Duration { + return 0 +} + +// SOCKS5ProxyURL implements tunnel.Tunnel +func (t *socks5Tunnel) SOCKS5ProxyURL() *url.URL { + return t.url +} + +// Stop implements tunnel.Tunnel +func (t *socks5Tunnel) Stop() { + // nothing +} + +// newTorOrTorsfTunnel creates a tor or torsf tunnel. +func newTorOrTorsfTunnel( + ctx context.Context, + logger model.Logger, + req *BootstrapRequest, + scheme string, +) (tunnel.Tunnel, error) { + config := &tunnel.Config{ + Name: scheme, + Session: &engine.SessionTunnelEarlySession{}, + SnowflakeRendezvous: req.SnowflakeRendezvousMethod, + TunnelDir: req.TunnelDir, + Logger: logger, + TorArgs: req.TorArgs, + TorBinary: req.TorBinary, + } + tun, _, err := tunnel.Start(ctx, config) + return tun, err +} + +// newPsiphonTunnel creates a psiphon tunnel. +func newPsiphonTunnel( + ctx context.Context, + logger model.Logger, + req *BootstrapRequest, +) (tunnel.Tunnel, error) { + config := &tunnel.Config{ + Name: "psiphon", + Session: &engine.SessionTunnelEarlySession{}, + SnowflakeRendezvous: "", // not needed + TunnelDir: req.TunnelDir, + Logger: logger, + TorArgs: nil, // not needed + TorBinary: "", // not needed + } + tun, _, err := tunnel.Start(ctx, config) + return tun, err +} diff --git a/internal/session/webconnectivity.go b/internal/session/webconnectivity.go new file mode 100644 index 0000000000..06d9297336 --- /dev/null +++ b/internal/session/webconnectivity.go @@ -0,0 +1,89 @@ +package session + +// +// Running the Web Connectivity experiment +// + +import ( + "context" + "time" + + "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivity" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// WebConnectivityRequest is a request to run Web Connectivity. +type WebConnectivityRequest struct { + // Input is the URL to measure using Web Connectivity. + Input string + + // ReportID is the report ID to use. + ReportID string + + // TestStartTime is when we started running this test. + TestStartTime time.Time +} + +// WebConnectivityEvent is emitted after we have run +// a measurement using Web Connectivity. +type WebConnectivityEvent struct { + // Error indicates a fundamental error occurred + // when running this experiment. + Error error + + // Measurement is the measurement result. + Measurement *model.Measurement +} + +// webconnectivity performs a measurement using Web Connectivity. +func (s *Session) webconnectivity(ctx context.Context, req *WebConnectivityRequest) { + runtimex.Assert(req != nil, "passed a nil req") + measurement, err := s.dowebconnectivity(ctx, req) + s.maybeEmit(&Event{ + WebConnectivity: &WebConnectivityEvent{ + Error: err, + Measurement: measurement, + }, + }) +} + +// dowebconnectivity implements webconnectivity. +func (s *Session) dowebconnectivity( + ctx context.Context, req *WebConnectivityRequest) (*model.Measurement, error) { + + if s.state.IsNone() { + return nil, ErrNotBootstrapped + } + + ts := newTickerService(ctx, s) + defer ts.stop() + + adapter, err := newSessionAdapter(s.state.Unwrap()) + if err != nil { + return nil, err + } + + cfg := webconnectivity.Config{} + runner := webconnectivity.NewExperimentMeasurer(cfg) + measurement := model.NewMeasurement( + adapter.location, + runner.ExperimentName(), + runner.ExperimentVersion(), + req.TestStartTime, + req.ReportID, + s.state.Unwrap().softwareName, + s.state.Unwrap().softwareVersion, + req.Input, + ) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(model.DiscardLogger), + Measurement: measurement, + Session: adapter, + } + + if err := runner.Run(ctx, args); err != nil { + return nil, err + } + return measurement, nil +} diff --git a/internal/sessionhttpclient/sessionhttpclient.go b/internal/sessionhttpclient/sessionhttpclient.go new file mode 100644 index 0000000000..b144b7187f --- /dev/null +++ b/internal/sessionhttpclient/sessionhttpclient.go @@ -0,0 +1,40 @@ +// Package sessionhttpclient creates an HTTP client for +// a measurement session. We will use this client for +// communicating with the OONI backend. +package sessionhttpclient + +import ( + "net/url" + + "github.com/ooni/probe-cli/v3/internal/bytecounter" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// Config contains config for creating a new session HTTP client. +type Config struct { + // ByteCounter is the MANDATORY byte counter to use. + ByteCounter *bytecounter.Counter + + // Logger is the MANDATORY logger to use. + Logger model.Logger + + // ProxyURL is the OPTIONAL proxy URL that the HTTPClient + // returned by New should be using. + ProxyURL *url.URL + + // Resolver is the MANDATORY resolver to use. + Resolver model.Resolver +} + +// New creates a new HTTPClient to be used during a measurement +// session to communicate with the OONI backend. +func New(config *Config) model.HTTPClient { + dialer := netxlite.NewDialerWithResolver(config.Logger, config.Resolver) + dialer = netxlite.MaybeWrapWithProxyDialer(dialer, config.ProxyURL) + handshaker := netxlite.NewTLSHandshakerStdlib(config.Logger) + tlsDialer := netxlite.NewTLSDialer(dialer, handshaker) + txp := netxlite.NewHTTPTransport(config.Logger, dialer, tlsDialer) + txp = bytecounter.MaybeWrapHTTPTransport(txp, config.ByteCounter) + return netxlite.NewHTTPClient(txp) +} diff --git a/internal/shellx/shellx.go b/internal/shellx/shellx.go index 8d09fbf62b..e47ad4c7fe 100644 --- a/internal/shellx/shellx.go +++ b/internal/shellx/shellx.go @@ -138,7 +138,7 @@ func cmd(config *Config, argv *Argv, envp *Envp) *execabs.Cmd { cmd.Env = append(cmd.Env, entry) } if config.Logger != nil { - cmdline := quotedCommandLineUnsafe(argv.P, argv.V...) + cmdline := QuotedCommandLineUnsafe(argv.P, argv.V...) config.Logger.Infof("+ %s", cmdline) } return cmd @@ -271,9 +271,9 @@ func OutputCommandLine(logger model.Logger, cmdline string) ([]byte, error) { // ErrNoCommandToExecute means that the command line is empty. var ErrNoCommandToExecute = errors.New("shellx: no command to execute") -// quotedCommandLineUnsafe returns a quoted command line. This function is unsafe +// QuotedCommandLineUnsafe returns a quoted command line. This function is unsafe // and SHOULD only be used to produce a nice output. -func quotedCommandLineUnsafe(command string, args ...string) string { +func QuotedCommandLineUnsafe(command string, args ...string) string { v := []string{} v = append(v, maybeQuoteArgUnsafe(command)) for _, a := range args { diff --git a/internal/tortunnel-old/config.go b/internal/tortunnel-old/config.go new file mode 100644 index 0000000000..456e842fb1 --- /dev/null +++ b/internal/tortunnel-old/config.go @@ -0,0 +1,81 @@ +package tortunnel + +import ( + "os" + + "github.com/ooni/probe-cli/v3/internal/model" +) + +// Config contains config for starting the tor tunnel. +type Config struct { + // BootstrapEvents is the OPTIONAL channel where to send + // events emitted during the bootstrap. + BootstrapEvents chan<- string `json:",omitempty"` + + // Dependencies is OPTIONAL and allow one to mock the functions + // called by Start, which is mainly useful for testing. + Dependencies *Dependencies `json:",omitempty"` + + // Logger is the OPTIONAL logger to use during the bootstrap. + Logger model.Logger `json:",omitempty"` + + // SnowflakeEnabled OPTIONALLY enables snowflake. + SnowflakeEnabled bool + + // SnowflakeRendezvousMethod is the OPTIONAL snowflake rendezvous method. + SnowflakeRendezvousMethod string + + // TunnelDir is the OPTIONAL directory in which to store state. + TunnelDir string + + // TorArgs contains OPTIONAL arguments to pass to tor. + TorArgs []string + + // TorBinary OPTIONAL tor binary path. + TorBinary string + + // TorVersion is the OPTIONAL channel where we send the version + // of tor that we are attempting to bootstrap. + TorVersion chan<- string `json:",omitempty"` +} + +// logger always returns a valid instance of [model.Logger]. +func (cc *Config) logger() model.Logger { + if cc.Logger == nil { + return model.DiscardLogger + } + return cc.Logger +} + +// dependencies always returns a valid instance of [Dependencies]. +func (cc *Config) dependencies() *Dependencies { + if cc.Dependencies == nil { + return defaultDependencies + } + return cc.Dependencies +} + +// tunnelDir returns the tunnel dir to use or an error. +func (cc *Config) tunnelDir(logger model.Logger) (string, func(), error) { + if cc.TunnelDir == "" { + dir, err := os.MkdirTemp("", "") + if err != nil { + return "", nil, err + } + return dir, cleanupTunnelDir(logger, dir), nil + } + return cc.TunnelDir, func() {}, nil +} + +// cleanupTunnelDir returns a function that removes a temporary tunnel dir. +func cleanupTunnelDir(logger model.Logger, dir string) func() { + return func() { + maybeRemoveDir(logger, dir) + } +} + +// maybeRemoveDir removes a directory if needed. +func maybeRemoveDir(logger model.Logger, dir string) { + logger.Infof("rm -rf %s", dir) + os.RemoveAll(dir) +} diff --git a/internal/tortunnel-old/dependencies.go b/internal/tortunnel-old/dependencies.go new file mode 100644 index 0000000000..1abf2d488d --- /dev/null +++ b/internal/tortunnel-old/dependencies.go @@ -0,0 +1,39 @@ +package tortunnel + +import ( + "context" + + "github.com/cretz/bine/control" + "github.com/cretz/bine/tor" +) + +// Dependencies contains dependencies allowing to mock [Start] for testing. +type Dependencies struct { + // Start should be equivalent to [tor.Start]. + Start func(ctx context.Context, conf *tor.StartConf) (*tor.Tor, error) + + // TorControlProtocolInfo should be equivalent to calling the + // ProtocolInfo method of instance.Control. + TorControlProtocolInfo func(instance *tor.Tor) (*control.ProtocolInfo, error) + + // TorControlGetInfo should be equivalent to calling the GetInfo + // method of the instance.Control. + TorControlGetInfo func(instance *tor.Tor, keys ...string) ([]*control.KeyVal, error) + + // TorEnableNetwork should be equivalent to calling the + // EnableNetwork method of instance. + TorEnableNetwork func(ctx context.Context, instance *tor.Tor, wait bool) error +} + +var defaultDependencies = &Dependencies{ + Start: tor.Start, + TorControlProtocolInfo: func(instance *tor.Tor) (*control.ProtocolInfo, error) { + return instance.Control.ProtocolInfo() + }, + TorControlGetInfo: func(instance *tor.Tor, keys ...string) ([]*control.KeyVal, error) { + return instance.Control.GetInfo(keys...) + }, + TorEnableNetwork: func(ctx context.Context, instance *tor.Tor, wait bool) error { + return instance.EnableNetwork(ctx, wait) + }, +} diff --git a/internal/tortunnel-old/desktop.go b/internal/tortunnel-old/desktop.go new file mode 100644 index 0000000000..fb556008f2 --- /dev/null +++ b/internal/tortunnel-old/desktop.go @@ -0,0 +1,41 @@ +//go:build !android && !ios && !ooni_libtor + +package tortunnel + +// +// This file implements our strategy for running tor on desktop in most +// configurations except for the ooni_libtor case, where we build tor and +// its dependencies for Linux. The purpuse of this special case it that +// of testing the otherwise untested code that would run on Android. +// + +import ( + "github.com/cretz/bine/tor" + "golang.org/x/sys/execabs" +) + +// newTorStartConf in this configuration uses torExePath to get a +// suitable tor binary and then executes it. +func newTorStartConf(config *Config, dataDir string, extraArgs []string) (*tor.StartConf, error) { + // determine the logget to use. + logger := config.logger() + + // determine the tor binary to use. + torBinary := config.TorBinary + if torBinary == "" { + var err error + torBinary, err = execabs.LookPath("tor") + if err != nil { + return nil, err + } + } + logger.Infof("tortunnel: using this tor binary: %s", torBinary) + + // generate and return the config. + tsc := &tor.StartConf{ + ExePath: torBinary, + DataDir: dataDir, + ExtraArgs: extraArgs, + } + return tsc, nil +} diff --git a/internal/tortunnel-old/doc.go b/internal/tortunnel-old/doc.go new file mode 100644 index 0000000000..6fde11a5c8 --- /dev/null +++ b/internal/tortunnel-old/doc.go @@ -0,0 +1,2 @@ +// Package tortunnel implements [model.Tunnel] using tor. +package tortunnel diff --git a/internal/tortunnel-old/start.go b/internal/tortunnel-old/start.go new file mode 100644 index 0000000000..7275a1883e --- /dev/null +++ b/internal/tortunnel-old/start.go @@ -0,0 +1,165 @@ +package tortunnel + +import ( + "context" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/cretz/bine/tor" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/shellx" +) + +// Start attempts to start a tor [Tunnel]. +func Start(ctx context.Context, config *Config) (*Tunnel, error) { + // obtain the logger to use + logger := config.logger() + + // determine the tunnel directory to use + tunnelDir, cleanupTunnelDir, err := config.tunnelDir(logger) + if err != nil { + return nil, err + } + stateDir := filepath.Join(tunnelDir, "tor") + logger.Infof("tortunnel: stateDir: %s", stateDir) + + // determine the log file to use + logFile := filepath.Join(stateDir, "tor.log") + logger.Infof("tortunnel: logFile: %s", logFile) + + // cleanup any leftovers from a previous invocation, if needed + maybeCleanupStateDir(logger, stateDir, logFile) + + // setup command line arguments. + extraArgs := append([]string{}, config.TorArgs...) + extraArgs = append(extraArgs, "Log") + extraArgs = append(extraArgs, fmt.Sprintf(`notice file %s`, logFile)) + logCommandLineArguments(logger, extraArgs) + + // TODO(bassosimone): handle snowflake + + // generate the start configuration + torStartConf, err := newTorStartConf(config, stateDir, extraArgs) + if err != nil { + cleanupTunnelDir() + return nil, err + } + + // obtain the dependencies we should use. + deps := config.dependencies() + + // start the tor process + instance, err := deps.Start(ctx, torStartConf) + if err != nil { + cleanupTunnelDir() + return nil, err + } + + // make sure we close the running process on Close + instance.StopProcessOnClose = true + + // obtain and emit the tor version + protoInfo, err := deps.TorControlProtocolInfo(instance) + if err != nil { + instance.Close() + cleanupTunnelDir() + return nil, err + } + logger.Infof("tortunnel: tor version: %s", protoInfo.TorVersion) + select { + case config.TorVersion <- protoInfo.TorVersion: + default: + } + + // wait for the bootstrap to complete + startBootstrap := time.Now() + if err := deps.TorEnableNetwork(ctx, instance, true); err != nil { + instance.Close() + cleanupTunnelDir() + return nil, err + } + bootstrapTime := time.Since(startBootstrap) + logger.Infof("tortunnel: bootstrap time: %v", bootstrapTime) + + // get the proxy URL + proxyURL, err := getProxyURL(deps, instance) + if err != nil { + instance.Close() + cleanupTunnelDir() + return nil, err + } + + // TODO(bassosimone): we still need to set the name correctly + + // construct a tunnel instance + tunnel := &Tunnel{ + bootstrapTime: bootstrapTime, + instance: instance, + maybeDeleteTunnelDir: cleanupTunnelDir, + name: "", + proxy: proxyURL, + stopOnce: sync.Once{}, + } + + return tunnel, nil +} + +// bootstrap performs the bootstrap and returns its duration. +func bootstrap(ctx context.Context, config *Config, instance *tor.Tor) (time.Duration, error) { +} + +// ErrCannotGetSOCKS5ProxyURL indicates we cannot get the SOCKS5 proxy URL. +var ErrCannotGetSOCKS5ProxyURL = errors.New("tortunnel: cannot get SOCKS5 proxy URL") + +// getProxyURL attempts to obtain the proxy URL. +func getProxyURL(deps *Dependencies, instance *tor.Tor) (*url.URL, error) { + // Adapted from + info, err := deps.TorControlGetInfo(instance, "net/listeners/socks") + if err != nil { + return nil, err + } + if len(info) != 1 || info[0].Key != "net/listeners/socks" { + instance.Close() + return nil, ErrCannotGetSOCKS5ProxyURL + } + proxyAddress := info[0].Val + if strings.HasPrefix(proxyAddress, "unix:") { + instance.Close() + return nil, ErrCannotGetSOCKS5ProxyURL + } + proxyURL := &url.URL{Scheme: "socks5", Host: proxyAddress} + return proxyURL, nil +} + +// maybeCleanupStateDir removes stale files inside the stateDir. +func maybeCleanupStateDir(logger model.Logger, stateDir, logFile string) { + maybeRemoveFile(logger, logFile) + maybeRemoveWithGlob(logger, filepath.Join(stateDir, "torrc-*")) + maybeRemoveWithGlob(logger, filepath.Join(stateDir, "control-port-*")) +} + +// maybeRemoveWithGlob globs and removes files. +func maybeRemoveWithGlob(logger model.Logger, pattern string) { + files, _ := filepath.Glob(pattern) + for _, file := range files { + maybeRemoveFile(logger, file) + } +} + +// maybeRemoveFile removes a file if needed. +func maybeRemoveFile(logger model.Logger, file string) { + logger.Infof("tortunnel: rm -f %s", file) + os.Remove(file) +} + +// logCommandLineArguments logs the command line arguments we're using. +func logCommandLineArguments(logger model.Logger, args []string) { + quoted := shellx.QuotedCommandLineUnsafe("tor", args...) + logger.Infof("tortunnel: command line: %s", quoted) +} diff --git a/internal/tortunnel-old/tunnel.go b/internal/tortunnel-old/tunnel.go new file mode 100644 index 0000000000..7687c1130f --- /dev/null +++ b/internal/tortunnel-old/tunnel.go @@ -0,0 +1,79 @@ +package tortunnel + +import ( + "context" + "net/url" + "sync" + "time" + + "github.com/cretz/bine/tor" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// Tunnel is a [model.Tunnel] implemented using tor. +type Tunnel struct { + // bootstrapTime is the duration of the bootstrap. + bootstrapTime time.Duration + + // instance is the running tor instance. + instance *tor.Tor + + // maybeDeleteTunnelDir is a cleanup function that deletes + // the tunnel dir if it's a temporary directory. + maybeDeleteTunnelDir func() + + // name is the tunnel name. + name string + + // proxy is the SOCKS5 proxy URL. + proxy *url.URL + + // stopOnce allows us to call stop just once. + stopOnce sync.Once +} + +var _ model.Tunnel = &Tunnel{} + +// BootstrapTime implements model.Tunnel +func (t *Tunnel) BootstrapTime() time.Duration { + return t.bootstrapTime +} + +// LookupProbeIP implements model.Tunnel +func (t *Tunnel) LookupProbeIP(ctx context.Context) (string, error) { + panic("not implemented") +} + +// Name implements model.Tunnel +func (t *Tunnel) Name() string { + return t.name +} + +// NewDNSOverHTTPSResolver implements model.Tunnel +func (t *Tunnel) NewDNSOverHTTPSResolver(logger model.Logger, URL string) model.Resolver { + httptxp := t.NewHTTPTransport(logger) + dnsxp := netxlite.NewDNSOverHTTPSTransportWithHTTPTransport(httptxp, URL) + return netxlite.WrapResolver(logger, netxlite.NewUnwrappedParallelResolver(dnsxp)) +} + +// NewHTTPTransport implements model.Tunnel +func (t *Tunnel) NewHTTPTransport(logger model.Logger) model.HTTPTransport { + dialer := netxlite.NewDialerWithoutResolver(logger) + dialer = netxlite.MaybeWrapWithProxyDialer(dialer, t.proxy) + tlsDialer := netxlite.NewTLSDialer(dialer, netxlite.NewTLSHandshakerStdlib(logger)) + return netxlite.NewHTTPTransport(logger, dialer, tlsDialer) +} + +// SOCKS5ProxyURL implements model.Tunnel +func (t *Tunnel) SOCKS5ProxyURL() *url.URL { + return t.proxy +} + +// Stop implements model.Tunnel +func (t *Tunnel) Stop() { + t.stopOnce.Do(func() { + t.instance.Close() + t.maybeDeleteTunnelDir() + }) +} diff --git a/internal/tortunnel/config.go b/internal/tortunnel/config.go new file mode 100644 index 0000000000..613692873e --- /dev/null +++ b/internal/tortunnel/config.go @@ -0,0 +1,85 @@ +package tortunnel + +// +// Config implementation +// + +import ( + "os" + + "github.com/ooni/probe-cli/v3/internal/model" +) + +// Config contains config for starting the tor tunnel. +type Config struct { + // BootstrapEvents is the OPTIONAL channel where to send + // events emitted during the bootstrap. + BootstrapEvents chan<- string `json:",omitempty"` + + // Dependencies is OPTIONAL and allow one to mock the functions + // called by Start, which is mainly useful for testing. + Dependencies *Dependencies `json:",omitempty"` + + // Logger is the OPTIONAL logger to use during the bootstrap. + Logger model.Logger `json:",omitempty"` + + // SnowflakeEnabled OPTIONALLY enables snowflake. + SnowflakeEnabled bool + + // SnowflakeRendezvousMethod is the OPTIONAL snowflake rendezvous method. + SnowflakeRendezvousMethod string + + // TunnelDir is the OPTIONAL directory in which to store state. + TunnelDir string + + // TorArgs contains OPTIONAL arguments to pass to tor. + TorArgs []string + + // TorBinary OPTIONAL tor binary path. + TorBinary string + + // TorVersion is the OPTIONAL channel where we send the version + // of tor that we are attempting to bootstrap. + TorVersion chan<- string `json:",omitempty"` +} + +// logger always returns a valid instance of [model.Logger]. +func (cc *Config) logger() model.Logger { + if cc.Logger == nil { + return model.DiscardLogger + } + return cc.Logger +} + +// dependencies always returns a valid instance of [Dependencies]. +func (cc *Config) dependencies() *Dependencies { + if cc.Dependencies == nil { + return defaultDependencies + } + return cc.Dependencies +} + +// tunnelDir returns the tunnel dir to use or an error. +func (cc *Config) tunnelDir(logger model.Logger) (string, func(), error) { + if cc.TunnelDir == "" { + dir, err := os.MkdirTemp("", "") + if err != nil { + return "", nil, err + } + return dir, cleanupTunnelDir(logger, dir), nil + } + return cc.TunnelDir, func() {}, nil +} + +// cleanupTunnelDir returns a function that removes a temporary tunnel dir. +func cleanupTunnelDir(logger model.Logger, dir string) func() { + return func() { + maybeRemoveDir(logger, dir) + } +} + +// maybeRemoveDir removes a directory if needed. +func maybeRemoveDir(logger model.Logger, dir string) { + logger.Infof("rm -rf %s", dir) + os.RemoveAll(dir) +} diff --git a/internal/tortunnel/dependencies.go b/internal/tortunnel/dependencies.go new file mode 100644 index 0000000000..21433379af --- /dev/null +++ b/internal/tortunnel/dependencies.go @@ -0,0 +1,43 @@ +package tortunnel + +// +// This package's dependencies (for testing). +// + +import ( + "context" + + "github.com/cretz/bine/control" + "github.com/cretz/bine/tor" +) + +// Dependencies contains dependencies allowing to mock [Start] for testing. +type Dependencies struct { + // Start should be equivalent to [tor.Start]. + Start func(ctx context.Context, conf *tor.StartConf) (*tor.Tor, error) + + // TorControlProtocolInfo should be equivalent to calling the + // ProtocolInfo method of instance.Control. + TorControlProtocolInfo func(instance *tor.Tor) (*control.ProtocolInfo, error) + + // TorControlGetInfo should be equivalent to calling the GetInfo + // method of the instance.Control. + TorControlGetInfo func(instance *tor.Tor, keys ...string) ([]*control.KeyVal, error) + + // TorEnableNetwork should be equivalent to calling the + // EnableNetwork method of instance. + TorEnableNetwork func(ctx context.Context, instance *tor.Tor, wait bool) error +} + +var defaultDependencies = &Dependencies{ + Start: tor.Start, + TorControlProtocolInfo: func(instance *tor.Tor) (*control.ProtocolInfo, error) { + return instance.Control.ProtocolInfo() + }, + TorControlGetInfo: func(instance *tor.Tor, keys ...string) ([]*control.KeyVal, error) { + return instance.Control.GetInfo(keys...) + }, + TorEnableNetwork: func(ctx context.Context, instance *tor.Tor, wait bool) error { + return instance.EnableNetwork(ctx, wait) + }, +} diff --git a/internal/tortunnel/doc.go b/internal/tortunnel/doc.go new file mode 100644 index 0000000000..001fc22ba5 --- /dev/null +++ b/internal/tortunnel/doc.go @@ -0,0 +1,2 @@ +// Package tortunnel allows to use tor as a [model.Tunnel]. +package tortunnel diff --git a/internal/tortunnel/tunnel.go b/internal/tortunnel/tunnel.go new file mode 100644 index 0000000000..f513429f68 --- /dev/null +++ b/internal/tortunnel/tunnel.go @@ -0,0 +1,7 @@ +package tortunnel + +// Tunnel is a [model.Tunnel] using tor. +type Tunnel struct{} + +// New creates a new instance of [Tunnel]. +func New() *Tunnel {} diff --git a/pkg/oonimkall/session.go b/pkg/oonimkall/session.go index 5ec01dc34d..b717a04aac 100644 --- a/pkg/oonimkall/session.go +++ b/pkg/oonimkall/session.go @@ -288,9 +288,9 @@ func (sess *Session) Geolocate(ctx *Context) (*GeolocateResults, error) { return nil, err } return &GeolocateResults{ - ASN: info.ASNString(), + ASN: info.ProbeASNString(), Country: info.CountryCode, - IP: info.ProbeIP, + IP: info.IPAddr, Org: info.NetworkName, }, nil } @@ -470,7 +470,7 @@ func (sess *Session) CheckIn(ctx *Context, config *CheckInConfig) (*CheckInInfo, Charging: config.Charging, OnWiFi: config.OnWiFi, Platform: config.Platform, - ProbeASN: info.ASNString(), + ProbeASN: info.ProbeASNString(), ProbeCC: info.CountryCode, RunType: model.RunType(config.RunType), SoftwareVersion: config.SoftwareVersion,