-
Notifications
You must be signed in to change notification settings - Fork 7
/
get_data.py
71 lines (54 loc) · 2.14 KB
/
get_data.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
import os
import requests
datasets = dict(
mnist=[
'http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz',
'http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz',
'http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz',
'http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz'
],
svhn=[
'http://ufldl.stanford.edu/housenumbers/train_32x32.mat',
'http://ufldl.stanford.edu/housenumbers/test_32x32.mat'
],
celeba={
'list_eval_partition.txt': '0B7EVK8r0v71pY0NSMzRuSXJEVkk',
'img_align_celeba.zip': '0B7EVK8r0v71pZjFTYXZWM3FlRnM'
}
)
data_dir = './data/'
if not os.path.exists(data_dir):
os.makedirs(data_dir)
def download_file_from_google_drive(id, destination):
def get_confirm_token(response):
for key, value in response.cookies.items():
if key.startswith('download_warning'):
return value
return None
def save_response_content(response, destination):
CHUNK_SIZE = 32768
with open(destination, "wb") as f:
for chunk in response.iter_content(CHUNK_SIZE):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
URL = "https://docs.google.com/uc?export=download"
session = requests.Session()
response = session.get(URL, params={'id': id}, stream=True)
token = get_confirm_token(response)
if token:
params = {'id': id, 'confirm': token}
response = session.get(URL, params=params, stream=True)
save_response_content(response, destination)
for key in datasets:
if not os.path.exists(data_dir + key):
os.makedirs(data_dir + key)
if isinstance(datasets[key], list):
for link in datasets[key]:
os.system('wget -nc %s -P %s' % (link, data_dir + str(key)))
else:
for filename in datasets[key]:
download_file_from_google_drive(
datasets[key][filename],
data_dir + str(key) + '/' + filename
)
os.system('cd %s/celeba && unzip img_align_celeba.zip && cd -' % data_dir)