-
Notifications
You must be signed in to change notification settings - Fork 0
/
index.js
80 lines (71 loc) · 2.5 KB
/
index.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
// Use node to run this code using the output of scraper.js
// This is downloading a lot of mp4 files.
// It will take a while depending on your internet speed
var http = require("https");
var fs = require("fs");
async function main() {
const rawdata = fs.readFileSync("scraped_links.json");
const sections = JSON.parse(rawdata);
console.log("Commencing downloads...")
for (var i = 0; i < sections.length; i++) {
const section = sections[i];
const sPath = `course_content/${section.sectionTitle}`;
fs.mkdirSync(sPath, { recursive: true });
await downloadSectionLectures(section, sPath);
console.log(
`Completed section ${i} of ${
sections.length - 1
}, moving to next...\n`
);
}
console.log("All downloads complete.\n");
}
async function downloadSectionLectures(section, sPath) {
const downloadableLectures = section.content.filter((item) =>
Boolean(item.downloadUrl)
);
await Promise.all(
downloadableLectures.map(
(lec) =>
new Promise((resolve) => {
const dest = `${sPath}/${lec.downloadFileName}`;
if (!fs.existsSync(dest))
download(lec.downloadUrl, dest).then(resolve);
else {
console.log(`${dest} already exists. Skipping...`);
resolve();
}
})
)
);
}
function clearIfExists(path) {
if (fs.existsSync(path)) fs.unlinkSync(path);
}
function download(downloadUrl, dest) {
const placeholderPath = `${dest}-incomplete`;
return new Promise((resolve) => {
// Remove any previous incomplete downloads
clearIfExists(placeholderPath);
// Create a new placeholder file
var file = fs.createWriteStream(placeholderPath);
try {
http.get(downloadUrl, function (response) {
response.pipe(file);
file.on("finish", function () {
file.close();
// The download has complete: remove the placeholder flag
fs.renameSync(placeholderPath, dest);
console.log(`Completed download for: ${dest}`);
resolve();
});
});
} catch (err) {
file.close();
console.log(`This download failed: ${dest}`);
console.log(err);
resolve();
}
});
}
main();