Merge pull request #13 from vierbergenlars/fix-issue-10

Regard all filesets with the same name as one fileset

dataaccess.go

@@ -26,18 +26,18 @@ type sqlQueries struct {
 
 var queries map[string]*sqlQueries = map[string]*sqlQueries{
 	"mysql": &sqlQueries{
-		JobList:               "SELECT j.Name, j.Type, j.ClientId, j.FileSetId, COALESCE(c.Name, ''), COALESCE(f.FileSet, ''), COUNT(*), SUM(j.JobBytes), SUM(j.JobFiles) FROM Job j LEFT JOIN Client c ON c.ClientId = j.ClientId LEFT JOIN FileSet f ON f.FileSetId = j.FileSetId GROUP BY j.Name, j.Type, j.ClientId, j.FileSetId, c.Name, f.FileSet HAVING MAX(j.SchedTime) >= ?",
-		LastJob:               "SELECT JobStatus,JobBytes,JobFiles,JobErrors,StartTime,COALESCE(EndTime, NOW()) FROM Job WHERE Name = ? AND ClientId = ? AND FileSetId = ? ORDER BY StartTime DESC LIMIT 1",
-		LastSuccessfulJob:     "SELECT JobStatus,JobBytes,JobFiles,JobErrors,StartTime,COALESCE(EndTime, NOW()) FROM Job WHERE Name = ? AND ClientId = ? AND FileSetId = ? AND JobStatus IN('T', 'W') ORDER BY StartTime DESC LIMIT 1",
-		LastSuccessfulFullJob: "SELECT JobStatus,JobBytes,JobFiles,JobErrors,StartTime,COALESCE(EndTime, NOW()) FROM Job WHERE Name = ? AND ClientId = ? AND FileSetId = ? AND JobStatus IN('T', 'W') AND Level = 'F' ORDER BY StartTime DESC LIMIT 1",
+		JobList:               "SELECT j.Name, j.Type, j.ClientId, COALESCE(c.Name, ''), COALESCE(f.FileSet, ''), COUNT(*), SUM(j.JobBytes), SUM(j.JobFiles) FROM Job j LEFT JOIN Client c ON c.ClientId = j.ClientId LEFT JOIN FileSet f ON f.FileSetId = j.FileSetId GROUP BY j.Name, j.Type, j.ClientId, c.Name, f.FileSet HAVING MAX(j.SchedTime) >= ?",
+		LastJob:               "SELECT JobStatus,JobBytes,JobFiles,JobErrors,StartTime,COALESCE(EndTime, NOW()) FROM Job WHERE Name = ? AND ClientId = ? AND FileSetId IN(SELECT f.FileSetId FROM FileSet f WHERE f.FileSet = ?) ORDER BY StartTime DESC LIMIT 1",
+		LastSuccessfulJob:     "SELECT JobStatus,JobBytes,JobFiles,JobErrors,StartTime,COALESCE(EndTime, NOW()) FROM Job WHERE Name = ? AND ClientId = ? AND FileSetId IN(SELECT f.FileSetId FROM FileSet f WHERE f.FileSet = ?) AND JobStatus IN('T', 'W') ORDER BY StartTime DESC LIMIT 1",
+		LastSuccessfulFullJob: "SELECT JobStatus,JobBytes,JobFiles,JobErrors,StartTime,COALESCE(EndTime, NOW()) FROM Job WHERE Name = ? AND ClientId = ? AND FileSetId IN(SELECT f.FileSetId FROM FileSet f WHERE f.FileSet = ?) AND JobStatus IN('T', 'W') AND Level = 'F' ORDER BY StartTime DESC LIMIT 1",
 		PoolInfo:              "SELECT p.name, sum(m.volbytes) AS bytes, count(*) AS volumes, (not exists(select * from JobMedia jm where jm.mediaid = m.mediaid)) AS prunable, COALESCE(TIMESTAMPADD(SECOND, m.volretention, m.lastwritten) < NOW(), false) AS expired FROM Media m LEFT JOIN Pool p ON m.poolid = p.poolid GROUP BY p.name, prunable, expired",
 		JobStates:             "SELECT JobStatus FROM Status",
 	},
 	"postgres": &sqlQueries{
-		JobList:               "SELECT j.Name, j.Type, j.ClientId, j.FileSetId, COALESCE(c.Name, ''), COALESCE(f.FileSet, ''), COUNT(*), SUM(j.JobBytes), SUM(j.JobFiles) FROM job j LEFT JOIN client c ON c.ClientId = j.ClientId LEFT JOIN fileset f ON f.FileSetId = j.FileSetId  GROUP BY j.Name, j.Type, j.ClientId, j.FileSetId, c.Name, f.FileSet HAVING MAX(j.SchedTime) >= $1",
-		LastJob:               "SELECT JobStatus,JobBytes,JobFiles,JobErrors,StartTime::timestamptz,COALESCE(EndTime::timestamptz, NOW()) FROM job WHERE Name = $1 AND ClientId = $2 AND FileSetId = $3 ORDER BY StartTime DESC LIMIT 1",
-		LastSuccessfulJob:     "SELECT JobStatus,JobBytes,JobFiles,JobErrors,StartTime::timestamptz,COALESCE(EndTime::timestamptz, NOW()) FROM job WHERE Name = $1 AND ClientId = $2 AND FileSetId = $3 AND JobStatus IN('T', 'W') ORDER BY StartTime DESC LIMIT 1",
-		LastSuccessfulFullJob: "SELECT JobStatus,JobBytes,JobFiles,JobErrors,StartTime::timestamptz,COALESCE(EndTime::timestamptz, NOW()) FROM job WHERE Name = $1 AND ClientId = $2 AND FileSetId = $3 AND JobStatus IN('T', 'W') AND Level = 'F' ORDER BY StartTime DESC LIMIT 1",
+		JobList:               "SELECT j.Name, j.Type, j.ClientId, COALESCE(c.Name, ''), COALESCE(f.FileSet, ''), COUNT(*), SUM(j.JobBytes), SUM(j.JobFiles) FROM job j LEFT JOIN client c ON c.ClientId = j.ClientId LEFT JOIN fileset f ON f.FileSetId = j.FileSetId  GROUP BY j.Name, j.Type, j.ClientId, c.Name, f.FileSet HAVING MAX(j.SchedTime) >= $1",
+		LastJob:               "SELECT JobStatus,JobBytes,JobFiles,JobErrors,StartTime::timestamptz,COALESCE(EndTime::timestamptz, NOW()) FROM job WHERE Name = $1 AND ClientId = $2 AND FileSetId IN(SELECT f.FileSetId from FileSet f WHERE f.FileSet = $3) ORDER BY StartTime DESC LIMIT 1",
+		LastSuccessfulJob:     "SELECT JobStatus,JobBytes,JobFiles,JobErrors,StartTime::timestamptz,COALESCE(EndTime::timestamptz, NOW()) FROM job WHERE Name = $1 AND ClientId = $2 AND FileSetId IN(SELECT f.FileSetId from FileSet f WHERE f.FileSet = $3) AND JobStatus IN('T', 'W') ORDER BY StartTime DESC LIMIT 1",
+		LastSuccessfulFullJob: "SELECT JobStatus,JobBytes,JobFiles,JobErrors,StartTime::timestamptz,COALESCE(EndTime::timestamptz, NOW()) FROM job WHERE Name = $1 AND ClientId = $2 AND FileSetId IN(SELECT f.FileSetId from FileSet f WHERE f.FileSet = $3) AND JobStatus IN('T', 'W') AND Level = 'F' ORDER BY StartTime DESC LIMIT 1",
 		PoolInfo:              "SELECT p.name, sum(m.volbytes) AS bytes, count(m) AS volumes, (not exists(select * from jobmedia jm where jm.mediaid = m.mediaid)) AS prunable, COALESCE((m.lastwritten + (m.volretention * interval '1s')) < NOW(), false) as expired FROM media m LEFT JOIN pool p ON m.poolid = p.poolid GROUP BY p.name, prunable, expired",
 		JobStates:             "SELECT JobStatus FROM status",
 	},
@@ -83,7 +83,7 @@ func (connection Connection) JobList() ([]JobInfo, error) {
 
 	for results.Next() {
 		var jobInfo JobInfo
-		err = results.Scan(&jobInfo.JobName, &jobInfo.JobType, &jobInfo.clientId, &jobInfo.fileSetId, &jobInfo.ClientName, &jobInfo.FileSetName, &jobInfo.TotalCount, &jobInfo.TotalBytes, &jobInfo.TotalFiles)
+		err = results.Scan(&jobInfo.JobName, &jobInfo.JobType, &jobInfo.clientId, &jobInfo.ClientName, &jobInfo.FileSetName, &jobInfo.TotalCount, &jobInfo.TotalBytes, &jobInfo.TotalFiles)
 		if err != nil {
 			return nil, err
 		}
@@ -105,7 +105,7 @@ func (connection Connection) execQuery(query string, args ...interface{}) (*sql.
 }
 
 func (connection Connection) execJobLookupQuery(query string, lookup *JobInfo) (*sql.Rows, error) {
-	return connection.execQuery(query, lookup.JobName, lookup.clientId, lookup.fileSetId)
+	return connection.execQuery(query, lookup.JobName, lookup.clientId, lookup.FileSetName)
 }
 
 func (connection Connection) execLastJobLookupQuery(query string, lookup *JobInfo) (*LastJob, error) {

development/mysql/03-issue-10.sql

@@ -0,0 +1,21 @@
+-- Reproduction of https://github.com/vierbergenlars/bareos_exporter/issues/10
+
+INSERT INTO FileSet (fileset, md5, createtime, filesettext)
+    VALUES
+        ('FileSetX', 'd41d8cd98f00b204e9800998ecf8427e', NOW() - interval 14 day, ''),
+        ('FileSetX', 'd41d8cd98f00b204e9800998ecf8427f', NOW() - interval 7 day, '');
+
+INSERT INTO Job (job, name, type, level, jobstatus, schedtime, starttime, poolid)
+    VALUES
+        ('cx-fsx.1', 'cx-fsx', 'B', 'F', 'T', NOW() - interval 13 day, NOW() - interval 13 day, (select poolid from Pool WHERE name = 'Pool1')),
+        ('cx-fsx.2', 'cx-fsx', 'B', 'F', 'T', NOW() - interval 6 day, NOW() - interval 6 day, (select poolid from Pool WHERE name = 'Pool1'));
+
+UPDATE Job j SET
+    clientid = (SELECT c.clientid from Client c WHERE c.name = 'c1-fd'),
+    filesetid = (SELECT f.filesetid from FileSet f WHERE f.fileset = 'FileSetX' AND md5 = 'd41d8cd98f00b204e9800998ecf8427e'),
+    endtime = j.starttime + interval 1 hour
+    WHERE j.name = 'cx-fsx';
+
+UPDATE Job j SET
+    filesetid = (SELECT f.filesetid from FileSet f WHERE f.fileset = 'FileSetX' AND md5 = 'd41d8cd98f00b204e9800998ecf8427f')
+    WHERE j.job = 'cx-fsx.2';

development/postgres/03-issue-10.sql

@@ -0,0 +1,21 @@
+-- Reproduction of https://github.com/vierbergenlars/bareos_exporter/issues/10
+
+INSERT INTO public.fileset (fileset, md5, createtime)
+    VALUES
+        ('FileSetX', 'd41d8cd98f00b204e9800998ecf8427e', NOW() - interval '2 weeks'),
+        ('FileSetX', 'd41d8cd98f00b204e9800998ecf8427f', NOW() - interval '1 week');
+
+INSERT INTO public.job (job, name, type, level, jobstatus, schedtime, starttime, poolid)
+    VALUES
+        ('cx-fsx.1', 'cx-fsx', 'B', 'F', 'T', NOW() - interval '1 week 6 days', NOW() - interval '1 week 6 days', (select poolid from pool WHERE name = 'Pool1')),
+        ('cx-fsx.2', 'cx-fsx', 'B', 'F', 'T', NOW() - interval '6 days', NOW() - interval '6 days', (select poolid from pool WHERE name = 'Pool1'));
+
+UPDATE public.job j SET
+    clientid = (SELECT c.clientid from public.client c WHERE c.name = 'c1-fd'),
+    filesetid = (SELECT f.filesetid from public.fileset f WHERE f.fileset = 'FileSetX' AND md5 = 'd41d8cd98f00b204e9800998ecf8427e'),
+    endtime = j.starttime + interval '1h'
+    WHERE j.name = 'cx-fsx';
+
+UPDATE public.job j SET
+    filesetid = (SELECT f.filesetid from public.fileset f WHERE f.fileset = 'FileSetX' AND md5 = 'd41d8cd98f00b204e9800998ecf8427f')
+    WHERE j.job = 'cx-fsx.2';

main.go

@@ -23,14 +23,6 @@ var (
 	jobDiscoveryDays = flag.Int("job-discovery-days", 7, "Number of days in the past that will be searched for jobs")
 )
 
-func init() {
-	flag.Usage = func() {
-		fmt.Println("Usage: bareos_exporter [ ... ]\n\nParameters:")
-		fmt.Println()
-		flag.PrintDefaults()
-	}
-}
-
 func splitDsn(dsn string) (string, string, error) {
 	var splitDsn = strings.SplitN(dsn, "://", 2)
 	if len(splitDsn) != 2 {
@@ -44,12 +36,12 @@ func main() {
 
 	dbType, connectionString, err := splitDsn(*databaseURL)
 	if err != nil {
-		panic(err.Error())
+		log.Fatal(err)
 	}
 
 	connection, err := GetConnection(dbType, connectionString, *jobDiscoveryDays)
 	if err != nil {
-		panic(err.Error())
+		log.Fatal(err)
 	}
 	defer connection.Close()
 	collector := bareosCollector(connection)

types.go

@@ -21,18 +21,17 @@ const (
 )
 
 type JobLookup struct {
-	JobName   string
-	clientId  int
-	fileSetId int
+	JobName     string
+	clientId    int
+	FileSetName string
 }
 
 // JobInfo models query results for static values of a job
 type JobInfo struct {
 	JobLookup
-	JobType     JobType
-	JobName     string
-	ClientName  string
-	FileSetName string
+	JobType    JobType
+	JobName    string
+	ClientName string
 
 	TotalCount int
 	TotalBytes int