Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

refactor(ooniprobe): remove deprecated parts #1004

Draft
wants to merge 4 commits into
base: master
Choose a base branch
from
Draft

refactor(ooniprobe): remove deprecated parts #1004

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
30bf24c
refactor(ooniprobe): begin removing deprecated parts
Dec 12, 2022
d5ff4b0
refactor(ooniprobe): remove testing functions
Dec 12, 2022
63aee49
refactor(ooniprobe): remove redundant conditional
Dec 12, 2022
4f86eab
refactor(ooniprobe): instantiate submitter before iterating inputs
Dec 12, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 53 additions & 30 deletions cmd/ooniprobe/internal/nettests/nettests.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,9 @@ type Controller struct {

// curInputIdx is the current input index
curInputIdx int

// saveToDisk indicates if we want to save the measurement to disk
saveToDisk bool
}

// BuildAndSetInputIdxMap takes in input a list of URLs in the format
Expand Down Expand Up @@ -141,17 +144,6 @@ func (c *Controller) Run(builder model.ExperimentBuilder, inputs []string) error
log.Debug(color.RedString("status.queued"))
log.Debug(color.RedString("status.started"))

if c.Probe.Config().Sharing.UploadResults {
if err := exp.OpenReportContext(context.Background()); err != nil {
log.Debugf(
"%s: %s", color.RedString("failure.report_create"), err.Error(),
)
} else {
log.Debugf(color.RedString("status.report_create"))
reportID = sql.NullString{String: exp.ReportID(), Valid: true}
}
}

maxRuntime := time.Duration(c.Probe.Config().Nettests.WebsitesMaxRuntime) * time.Second
if c.RunType == model.RunTypeTimed && maxRuntime > 0 {
log.Debug("disabling maxRuntime when running in the background")
Expand All @@ -168,6 +160,13 @@ func (c *Controller) Run(builder model.ExperimentBuilder, inputs []string) error
}
start := time.Now()
c.ntStartTime = start

ctx := context.Background()
submitter, err := c.newSubmitter(ctx)
if err != nil {
return errors.Wrap(err, "failed to initialise submitter")
}

for idx, input := range inputs {
if c.Probe.IsTerminated() {
log.Info("user requested us to terminate using Ctrl-C")
Expand All @@ -185,6 +184,10 @@ func (c *Controller) Run(builder model.ExperimentBuilder, inputs []string) error
urlID = sql.NullInt64{Int64: c.inputIdxMap[idx64], Valid: true}
}

// TODO(DecFox): we currently pass a nil reportID which should be replaced by the reportID
// we get from the submitter. However, since the reportID is generated on uploading the first
// measurement, we do not have a valid reportID to pass here. Therefore, we want to populate the
// submitter without having to upload a measurement.
msmt, err := db.CreateMeasurement(
reportID, exp.Name(), c.res.MeasurementDir, idx, resultID, urlID,
)
Expand Down Expand Up @@ -212,28 +215,29 @@ func (c *Controller) Run(builder model.ExperimentBuilder, inputs []string) error
continue
}

saveToDisk := true
if c.Probe.Config().Sharing.UploadResults {
// Implementation note: SubmitMeasurement will fail here if we did fail
// to open the report but we still want to continue. There will be a
// bit of a spew in the logs, perhaps, but stopping seems less efficient.
if err := exp.SubmitAndUpdateMeasurementContext(context.Background(), measurement); err != nil {
log.Debug(color.RedString("failure.measurement_submission"))
if err := db.UploadFailed(c.msmts[idx64], err.Error()); err != nil {
return errors.Wrap(err, "failed to mark upload as failed")
}
} else if err := db.UploadSucceeded(c.msmts[idx64]); err != nil {
return errors.Wrap(err, "failed to mark upload as succeeded")
} else {
// Everything went OK, don't save to disk
saveToDisk = false
c.saveToDisk = true

// upload measurement
if err := submitter.Submit(ctx, measurement); err != nil {
log.Debug(color.RedString("failure.measurement_submission"))
if err := db.UploadFailed(c.msmts[idx64], err.Error()); err != nil {
return errors.Wrap(err, "failed to mark upload as failed")
}
} else if err := db.UploadSucceeded(c.msmts[idx64]); err != nil {
return errors.Wrap(err, "failed to mark upload as succeeded")
} else {
// Everything went OK, don't save to disk
c.saveToDisk = false
}

// We only save the measurement to disk if we failed to upload the measurement
if saveToDisk {
if err := exp.SaveMeasurement(measurement, msmt.MeasurementFilePath.String); err != nil {
return errors.Wrap(err, "failed to save measurement on disk")
}
saver, err := c.newSaver(exp, *msmt)
if err != nil {
return errors.Wrap(err, "failed to initialise saver")
}
err = saver.SaveMeasurement(measurement)
if err != nil {
return errors.Wrap(err, "failed to save measurement on disk")
}

if err := db.Done(c.msmts[idx64]); err != nil {
Expand Down Expand Up @@ -295,3 +299,22 @@ func (c *Controller) OnProgress(perc float64, msg string) {
key := fmt.Sprintf("%T", c.nt)
output.Progress(key, perc, eta, msg)
}

// newSaver creates a new engine.Saver instance.
func (c *Controller) newSaver(experiment model.Experiment, msmt model.DatabaseMeasurement) (engine.Saver, error) {
return engine.NewSaver(engine.SaverConfig{
Enabled: c.saveToDisk,
Experiment: experiment,
FilePath: msmt.MeasurementFilePath.String,
Logger: c.Session.Logger(),
})
}

// newSubmitter creates a new engine.Submitter instance.
func (c *Controller) newSubmitter(ctx context.Context) (engine.Submitter, error) {
return engine.NewSubmitter(ctx, engine.SubmitterConfig{
Enabled: c.Probe.Config().Sharing.UploadResults,
Session: c.Session,
Logger: c.Session.Logger(),
})
}